From dc06e9df02ff78411f0533609b93c85be6a93494 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Mon, 14 Feb 2022 13:11:49 +0100 Subject: [PATCH 001/600] move predicates that depend on isReDoSCandidate into a ReDoSPruning module --- .../java/security/performance/ReDoSUtil.qll | 630 +++++++++--------- .../performance/SuperlinearBackTracking.qll | 2 +- .../query-tests/security/CWE-730/ReDoS.ql | 2 +- .../security/performance/ReDoSUtil.qll | 630 +++++++++--------- .../performance/SuperlinearBackTracking.qll | 2 +- javascript/ql/src/Performance/ReDoS.ql | 2 +- .../python/security/performance/ReDoSUtil.qll | 630 +++++++++--------- .../performance/SuperlinearBackTracking.qll | 2 +- python/ql/src/Security/CWE-730/ReDoS.ql | 2 +- .../ruby/security/performance/ReDoSUtil.qll | 630 +++++++++--------- .../performance/SuperlinearBackTracking.qll | 2 +- .../ql/src/queries/security/cwe-1333/ReDoS.ql | 2 +- 12 files changed, 1272 insertions(+), 1264 deletions(-) diff --git a/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll b/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll index 8aa348bf62f..8315d2b8225 100644 --- a/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll +++ b/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll @@ -14,45 +14,6 @@ import ReDoSUtilSpecific -/** - * A configuration for which parts of a regular expression should be considered relevant for - * the different predicates in `ReDoS.qll`. - * Used to adjust the computations for either superlinear or exponential backtracking. - */ -abstract class ReDoSConfiguration extends string { - bindingset[this] - ReDoSConfiguration() { any() } - - /** - * Holds if `state` with the pump string `pump` is a candidate for a - * ReDoS vulnerable state. - * This is used to determine which states are considered for the prefix/suffix construction. - */ - abstract predicate isReDoSCandidate(State state, string pump); -} - -/** - * Holds if repeating `pump` starting at `state` is a candidate for causing backtracking. - * No check whether a rejected suffix exists has been made. - */ -private predicate isReDoSCandidate(State state, string pump) { - any(ReDoSConfiguration conf).isReDoSCandidate(state, pump) and - ( - not any(ReDoSConfiguration conf).isReDoSCandidate(epsilonSucc+(state), _) - or - epsilonSucc+(state) = state and - state = - max(State s, Location l | - s = epsilonSucc+(state) and - l = s.getRepr().getLocation() and - any(ReDoSConfiguration conf).isReDoSCandidate(s, _) and - s.getRepr() instanceof InfiniteRepetitionQuantifier - | - s order by l.getStartLine(), l.getStartColumn(), l.getEndColumn(), l.getEndLine() - ) - ) -} - /** * Gets the char after `c` (from a simplified ASCII table). */ @@ -877,266 +838,341 @@ predicate isStartState(State state) { } /** - * Predicates for constructing a prefix string that leads to a given state. + * A configuration for which parts of a regular expression should be considered relevant for + * the different predicates in `ReDoS.qll`. + * Used to adjust the computations for either superlinear or exponential backtracking. */ -private module PrefixConstruction { - /** - * Holds if `state` is the textually last start state for the regular expression. - */ - private predicate lastStartState(State state) { - exists(RegExpRoot root | - state = - max(StateInPumpableRegexp s, Location l | - isStartState(s) and getRoot(s.getRepr()) = root and l = s.getRepr().getLocation() - | - s - order by - l.getStartLine(), l.getStartColumn(), s.getRepr().toString(), l.getEndColumn(), - l.getEndLine() - ) - ) - } +abstract class ReDoSConfiguration extends string { + bindingset[this] + ReDoSConfiguration() { any() } /** - * Holds if there exists any transition (Epsilon() or other) from `a` to `b`. + * Holds if `state` with the pump string `pump` is a candidate for a + * ReDoS vulnerable state. + * This is used to determine which states are considered for the prefix/suffix construction. */ - private predicate existsTransition(State a, State b) { delta(a, _, b) } - - /** - * Gets the minimum number of transitions it takes to reach `state` from the `start` state. - */ - int prefixLength(State start, State state) = - shortestDistances(lastStartState/1, existsTransition/2)(start, state, result) - - /** - * Gets the minimum number of transitions it takes to reach `state` from the start state. - */ - private int lengthFromStart(State state) { result = prefixLength(_, state) } - - /** - * Gets a string for which the regular expression will reach `state`. - * - * Has at most one result for any given `state`. - * This predicate will not always have a result even if there is a ReDoS issue in - * the regular expression. - */ - string prefix(State state) { - lastStartState(state) and - result = "" - or - // the search stops past the last redos candidate state. - lengthFromStart(state) <= max(lengthFromStart(any(State s | isReDoSCandidate(s, _)))) and - exists(State prev | - // select a unique predecessor (by an arbitrary measure) - prev = - min(State s, Location loc | - lengthFromStart(s) = lengthFromStart(state) - 1 and - loc = s.getRepr().getLocation() and - delta(s, _, state) - | - s - order by - loc.getStartLine(), loc.getStartColumn(), loc.getEndLine(), loc.getEndColumn(), - s.getRepr().toString() - ) - | - // greedy search for the shortest prefix - result = prefix(prev) and delta(prev, Epsilon(), state) - or - not delta(prev, Epsilon(), state) and - result = prefix(prev) + getCanonicalEdgeChar(prev, state) - ) - } - - /** - * Gets a canonical char for which there exists a transition from `prev` to `next` in the NFA. - */ - private string getCanonicalEdgeChar(State prev, State next) { - result = - min(string c | delta(prev, any(InputSymbol symbol | c = intersect(Any(), symbol)), next)) - } - - /** - * A state within a regular expression that has a pumpable state. - */ - class StateInPumpableRegexp extends State { - pragma[noinline] - StateInPumpableRegexp() { - exists(State s | isReDoSCandidate(s, _) | getRoot(s.getRepr()) = getRoot(this.getRepr())) - } - } + abstract predicate isReDoSCandidate(State state, string pump); } -/** - * Predicates for testing the presence of a rejecting suffix. - * - * These predicates are used to ensure that the all states reached from the fork - * by repeating `w` have a rejecting suffix. - * - * For example, a regexp like `/^(a+)+/` will accept any string as long the prefix is - * some number of `"a"`s, and it is therefore not possible to construct a rejecting suffix. - * - * A regexp like `/(a+)+$/` or `/(a+)+b/` trivially has a rejecting suffix, - * as the suffix "X" will cause both the regular expressions to be rejected. - * - * The string `w` is repeated any number of times because it needs to be - * infinitely repeatedable for the attack to work. - * For the regular expression `/((ab)+)*abab/` the accepting state is not reachable from the fork - * using epsilon transitions. But any attempt at repeating `w` will end in a state that accepts all suffixes. - */ -private module SuffixConstruction { - import PrefixConstruction - +module ReDoSPruning { /** - * Holds if all states reachable from `fork` by repeating `w` - * are likely rejectable by appending some suffix. + * Holds if repeating `pump` starting at `state` is a candidate for causing backtracking. + * No check whether a rejected suffix exists has been made. */ - predicate reachesOnlyRejectableSuffixes(State fork, string w) { - isReDoSCandidate(fork, w) and - forex(State next | next = process(fork, w, w.length() - 1) | isLikelyRejectable(next)) - } - - /** - * Holds if there likely exists a suffix starting from `s` that leads to the regular expression being rejected. - * This predicate might find impossible suffixes when searching for suffixes of length > 1, which can cause FPs. - */ - pragma[noinline] - private predicate isLikelyRejectable(StateInPumpableRegexp s) { - // exists a reject edge with some char. - hasRejectEdge(s) - or - hasEdgeToLikelyRejectable(s) - or - // stopping here is rejection - isRejectState(s) - } - - /** - * Holds if `s` is not an accept state, and there is no epsilon transition to an accept state. - */ - predicate isRejectState(StateInPumpableRegexp s) { not epsilonSucc*(s) = Accept(_) } - - /** - * Holds if there is likely a non-empty suffix leading to rejection starting in `s`. - */ - pragma[noopt] - predicate hasEdgeToLikelyRejectable(StateInPumpableRegexp s) { - // all edges (at least one) with some char leads to another state that is rejectable. - // the `next` states might not share a common suffix, which can cause FPs. - exists(string char | char = hasEdgeToLikelyRejectableHelper(s) | - // noopt to force `hasEdgeToLikelyRejectableHelper` to be first in the join-order. - exists(State next | deltaClosedChar(s, char, next) | isLikelyRejectable(next)) and - forall(State next | deltaClosedChar(s, char, next) | isLikelyRejectable(next)) - ) - } - - /** - * Gets a char for there exists a transition away from `s`, - * and `s` has not been found to be rejectable by `hasRejectEdge` or `isRejectState`. - */ - pragma[noinline] - private string hasEdgeToLikelyRejectableHelper(StateInPumpableRegexp s) { - not hasRejectEdge(s) and - not isRejectState(s) and - deltaClosedChar(s, result, _) - } - - /** - * Holds if there is a state `next` that can be reached from `prev` - * along epsilon edges, such that there is a transition from - * `prev` to `next` that the character symbol `char`. - */ - predicate deltaClosedChar(StateInPumpableRegexp prev, string char, StateInPumpableRegexp next) { - deltaClosed(prev, getAnInputSymbolMatchingRelevant(char), next) - } - - pragma[noinline] - InputSymbol getAnInputSymbolMatchingRelevant(string char) { - char = relevant(_) and - result = getAnInputSymbolMatching(char) - } - - /** - * Gets a char used for finding possible suffixes inside `root`. - */ - pragma[noinline] - private string relevant(RegExpRoot root) { - exists(ascii(result)) and exists(root) - or - exists(InputSymbol s | belongsTo(s, root) | result = intersect(s, _)) - or - // The characters from `hasSimpleRejectEdge`. Only `\n` is really needed (as `\n` is not in the `ascii` relation). - // The three chars must be kept in sync with `hasSimpleRejectEdge`. - result = ["|", "\n", "Z"] and exists(root) - } - - /** - * Holds if there exists a `char` such that there is no edge from `s` labeled `char` in our NFA. - * The NFA does not model reject states, so the above is the same as saying there is a reject edge. - */ - private predicate hasRejectEdge(State s) { - hasSimpleRejectEdge(s) - or - not hasSimpleRejectEdge(s) and - exists(string char | char = relevant(getRoot(s.getRepr())) | not deltaClosedChar(s, char, _)) - } - - /** - * Holds if there is no edge from `s` labeled with "|", "\n", or "Z" in our NFA. - * This predicate is used as a cheap pre-processing to speed up `hasRejectEdge`. - */ - private predicate hasSimpleRejectEdge(State s) { - // The three chars were chosen arbitrarily. The three chars must be kept in sync with `relevant`. - exists(string char | char = ["|", "\n", "Z"] | not deltaClosedChar(s, char, _)) - } - - /** - * Gets a state that can be reached from pumpable `fork` consuming all - * chars in `w` any number of times followed by the first `i+1` characters of `w`. - */ - pragma[noopt] - private State process(State fork, string w, int i) { - exists(State prev | prev = getProcessPrevious(fork, i, w) | - exists(string char, InputSymbol sym | - char = w.charAt(i) and - deltaClosed(prev, sym, result) and - // noopt to prevent joining `prev` with all possible `chars` that could transition away from `prev`. - // Instead only join with the set of `chars` where a relevant `InputSymbol` has already been found. - sym = getAProcessInputSymbol(char) - ) - ) - } - - /** - * Gets a state that can be reached from pumpable `fork` consuming all - * chars in `w` any number of times followed by the first `i` characters of `w`. - */ - private State getProcessPrevious(State fork, int i, string w) { - isReDoSCandidate(fork, w) and + private predicate isReDoSCandidate(State state, string pump) { + any(ReDoSConfiguration conf).isReDoSCandidate(state, pump) and ( - i = 0 and result = fork + not any(ReDoSConfiguration conf).isReDoSCandidate(epsilonSucc+(state), _) or - result = process(fork, w, i - 1) - or - // repeat until fixpoint - i = 0 and - result = process(fork, w, w.length() - 1) + epsilonSucc+(state) = state and + state = + max(State s, Location l | + s = epsilonSucc+(state) and + l = s.getRepr().getLocation() and + any(ReDoSConfiguration conf).isReDoSCandidate(s, _) and + s.getRepr() instanceof InfiniteRepetitionQuantifier + | + s order by l.getStartLine(), l.getStartColumn(), l.getEndColumn(), l.getEndLine() + ) ) } /** - * Gets an InputSymbol that matches `char`. - * The predicate is specialized to only have a result for the `char`s that are relevant for the `process` predicate. + * Predicates for constructing a prefix string that leads to a given state. */ - private InputSymbol getAProcessInputSymbol(string char) { - char = getAProcessChar() and - result = getAnInputSymbolMatching(char) + private module PrefixConstruction { + /** + * Holds if `state` is the textually last start state for the regular expression. + */ + private predicate lastStartState(State state) { + exists(RegExpRoot root | + state = + max(StateInPumpableRegexp s, Location l | + isStartState(s) and getRoot(s.getRepr()) = root and l = s.getRepr().getLocation() + | + s + order by + l.getStartLine(), l.getStartColumn(), s.getRepr().toString(), l.getEndColumn(), + l.getEndLine() + ) + ) + } + + /** + * Holds if there exists any transition (Epsilon() or other) from `a` to `b`. + */ + private predicate existsTransition(State a, State b) { delta(a, _, b) } + + /** + * Gets the minimum number of transitions it takes to reach `state` from the `start` state. + */ + int prefixLength(State start, State state) = + shortestDistances(lastStartState/1, existsTransition/2)(start, state, result) + + /** + * Gets the minimum number of transitions it takes to reach `state` from the start state. + */ + private int lengthFromStart(State state) { result = prefixLength(_, state) } + + /** + * Gets a string for which the regular expression will reach `state`. + * + * Has at most one result for any given `state`. + * This predicate will not always have a result even if there is a ReDoS issue in + * the regular expression. + */ + string prefix(State state) { + lastStartState(state) and + result = "" + or + // the search stops past the last redos candidate state. + lengthFromStart(state) <= max(lengthFromStart(any(State s | isReDoSCandidate(s, _)))) and + exists(State prev | + // select a unique predecessor (by an arbitrary measure) + prev = + min(State s, Location loc | + lengthFromStart(s) = lengthFromStart(state) - 1 and + loc = s.getRepr().getLocation() and + delta(s, _, state) + | + s + order by + loc.getStartLine(), loc.getStartColumn(), loc.getEndLine(), loc.getEndColumn(), + s.getRepr().toString() + ) + | + // greedy search for the shortest prefix + result = prefix(prev) and delta(prev, Epsilon(), state) + or + not delta(prev, Epsilon(), state) and + result = prefix(prev) + getCanonicalEdgeChar(prev, state) + ) + } + + /** + * Gets a canonical char for which there exists a transition from `prev` to `next` in the NFA. + */ + private string getCanonicalEdgeChar(State prev, State next) { + result = + min(string c | delta(prev, any(InputSymbol symbol | c = intersect(Any(), symbol)), next)) + } + + /** + * A state within a regular expression that has a pumpable state. + */ + class StateInPumpableRegexp extends State { + pragma[noinline] + StateInPumpableRegexp() { + exists(State s | isReDoSCandidate(s, _) | getRoot(s.getRepr()) = getRoot(this.getRepr())) + } + } } /** - * Gets a `char` that occurs in a `pump` string. + * Predicates for testing the presence of a rejecting suffix. + * + * These predicates are used to ensure that the all states reached from the fork + * by repeating `w` have a rejecting suffix. + * + * For example, a regexp like `/^(a+)+/` will accept any string as long the prefix is + * some number of `"a"`s, and it is therefore not possible to construct a rejecting suffix. + * + * A regexp like `/(a+)+$/` or `/(a+)+b/` trivially has a rejecting suffix, + * as the suffix "X" will cause both the regular expressions to be rejected. + * + * The string `w` is repeated any number of times because it needs to be + * infinitely repeatedable for the attack to work. + * For the regular expression `/((ab)+)*abab/` the accepting state is not reachable from the fork + * using epsilon transitions. But any attempt at repeating `w` will end in a state that accepts all suffixes. */ - private string getAProcessChar() { result = any(string s | isReDoSCandidate(_, s)).charAt(_) } + private module SuffixConstruction { + import PrefixConstruction + + /** + * Holds if all states reachable from `fork` by repeating `w` + * are likely rejectable by appending some suffix. + */ + predicate reachesOnlyRejectableSuffixes(State fork, string w) { + isReDoSCandidate(fork, w) and + forex(State next | next = process(fork, w, w.length() - 1) | isLikelyRejectable(next)) + } + + /** + * Holds if there likely exists a suffix starting from `s` that leads to the regular expression being rejected. + * This predicate might find impossible suffixes when searching for suffixes of length > 1, which can cause FPs. + */ + pragma[noinline] + private predicate isLikelyRejectable(StateInPumpableRegexp s) { + // exists a reject edge with some char. + hasRejectEdge(s) + or + hasEdgeToLikelyRejectable(s) + or + // stopping here is rejection + isRejectState(s) + } + + /** + * Holds if `s` is not an accept state, and there is no epsilon transition to an accept state. + */ + predicate isRejectState(StateInPumpableRegexp s) { not epsilonSucc*(s) = Accept(_) } + + /** + * Holds if there is likely a non-empty suffix leading to rejection starting in `s`. + */ + pragma[noopt] + predicate hasEdgeToLikelyRejectable(StateInPumpableRegexp s) { + // all edges (at least one) with some char leads to another state that is rejectable. + // the `next` states might not share a common suffix, which can cause FPs. + exists(string char | char = hasEdgeToLikelyRejectableHelper(s) | + // noopt to force `hasEdgeToLikelyRejectableHelper` to be first in the join-order. + exists(State next | deltaClosedChar(s, char, next) | isLikelyRejectable(next)) and + forall(State next | deltaClosedChar(s, char, next) | isLikelyRejectable(next)) + ) + } + + /** + * Gets a char for there exists a transition away from `s`, + * and `s` has not been found to be rejectable by `hasRejectEdge` or `isRejectState`. + */ + pragma[noinline] + private string hasEdgeToLikelyRejectableHelper(StateInPumpableRegexp s) { + not hasRejectEdge(s) and + not isRejectState(s) and + deltaClosedChar(s, result, _) + } + + /** + * Holds if there is a state `next` that can be reached from `prev` + * along epsilon edges, such that there is a transition from + * `prev` to `next` that the character symbol `char`. + */ + predicate deltaClosedChar(StateInPumpableRegexp prev, string char, StateInPumpableRegexp next) { + deltaClosed(prev, getAnInputSymbolMatchingRelevant(char), next) + } + + pragma[noinline] + InputSymbol getAnInputSymbolMatchingRelevant(string char) { + char = relevant(_) and + result = getAnInputSymbolMatching(char) + } + + /** + * Gets a char used for finding possible suffixes inside `root`. + */ + pragma[noinline] + private string relevant(RegExpRoot root) { + exists(ascii(result)) and exists(root) + or + exists(InputSymbol s | belongsTo(s, root) | result = intersect(s, _)) + or + // The characters from `hasSimpleRejectEdge`. Only `\n` is really needed (as `\n` is not in the `ascii` relation). + // The three chars must be kept in sync with `hasSimpleRejectEdge`. + result = ["|", "\n", "Z"] and exists(root) + } + + /** + * Holds if there exists a `char` such that there is no edge from `s` labeled `char` in our NFA. + * The NFA does not model reject states, so the above is the same as saying there is a reject edge. + */ + private predicate hasRejectEdge(State s) { + hasSimpleRejectEdge(s) + or + not hasSimpleRejectEdge(s) and + exists(string char | char = relevant(getRoot(s.getRepr())) | not deltaClosedChar(s, char, _)) + } + + /** + * Holds if there is no edge from `s` labeled with "|", "\n", or "Z" in our NFA. + * This predicate is used as a cheap pre-processing to speed up `hasRejectEdge`. + */ + private predicate hasSimpleRejectEdge(State s) { + // The three chars were chosen arbitrarily. The three chars must be kept in sync with `relevant`. + exists(string char | char = ["|", "\n", "Z"] | not deltaClosedChar(s, char, _)) + } + + /** + * Gets a state that can be reached from pumpable `fork` consuming all + * chars in `w` any number of times followed by the first `i+1` characters of `w`. + */ + pragma[noopt] + private State process(State fork, string w, int i) { + exists(State prev | prev = getProcessPrevious(fork, i, w) | + exists(string char, InputSymbol sym | + char = w.charAt(i) and + deltaClosed(prev, sym, result) and + // noopt to prevent joining `prev` with all possible `chars` that could transition away from `prev`. + // Instead only join with the set of `chars` where a relevant `InputSymbol` has already been found. + sym = getAProcessInputSymbol(char) + ) + ) + } + + /** + * Gets a state that can be reached from pumpable `fork` consuming all + * chars in `w` any number of times followed by the first `i` characters of `w`. + */ + private State getProcessPrevious(State fork, int i, string w) { + isReDoSCandidate(fork, w) and + ( + i = 0 and result = fork + or + result = process(fork, w, i - 1) + or + // repeat until fixpoint + i = 0 and + result = process(fork, w, w.length() - 1) + ) + } + + /** + * Gets an InputSymbol that matches `char`. + * The predicate is specialized to only have a result for the `char`s that are relevant for the `process` predicate. + */ + private InputSymbol getAProcessInputSymbol(string char) { + char = getAProcessChar() and + result = getAnInputSymbolMatching(char) + } + + /** + * Gets a `char` that occurs in a `pump` string. + */ + private string getAProcessChar() { result = any(string s | isReDoSCandidate(_, s)).charAt(_) } + } + + /** + * Holds if `term` may cause superlinear backtracking on strings containing many repetitions of `pump`. + * Gets the shortest string that causes superlinear backtracking. + */ + private predicate isReDoSAttackable(RegExpTerm term, string pump, State s) { + exists(int i, string c | s = Match(term, i) | + c = + min(string w | + any(ReDoSConfiguration conf).isReDoSCandidate(s, w) and + SuffixConstruction::reachesOnlyRejectableSuffixes(s, w) + | + w order by w.length(), w + ) and + pump = escape(rotate(c, i)) + ) + } + + /** + * Holds if the state `s` (represented by the term `t`) can have backtracking with repetitions of `pump`. + * + * `prefixMsg` contains a friendly message for a prefix that reaches `s` (or `prefixMsg` is the empty string if the prefix is empty or if no prefix could be found). + */ + predicate hasReDoSResult(RegExpTerm t, string pump, State s, string prefixMsg) { + isReDoSAttackable(t, pump, s) and + ( + prefixMsg = "starting with '" + escape(PrefixConstruction::prefix(s)) + "' and " and + not PrefixConstruction::prefix(s) = "" + or + PrefixConstruction::prefix(s) = "" and prefixMsg = "" + or + not exists(PrefixConstruction::prefix(s)) and prefixMsg = "" + ) + } } /** @@ -1162,37 +1198,3 @@ bindingset[str, i] private string rotate(string str, int i) { result = str.suffix(str.length() - i) + str.prefix(str.length() - i) } - -/** - * Holds if `term` may cause superlinear backtracking on strings containing many repetitions of `pump`. - * Gets the shortest string that causes superlinear backtracking. - */ -private predicate isReDoSAttackable(RegExpTerm term, string pump, State s) { - exists(int i, string c | s = Match(term, i) | - c = - min(string w | - any(ReDoSConfiguration conf).isReDoSCandidate(s, w) and - SuffixConstruction::reachesOnlyRejectableSuffixes(s, w) - | - w order by w.length(), w - ) and - pump = escape(rotate(c, i)) - ) -} - -/** - * Holds if the state `s` (represented by the term `t`) can have backtracking with repetitions of `pump`. - * - * `prefixMsg` contains a friendly message for a prefix that reaches `s` (or `prefixMsg` is the empty string if the prefix is empty or if no prefix could be found). - */ -predicate hasReDoSResult(RegExpTerm t, string pump, State s, string prefixMsg) { - isReDoSAttackable(t, pump, s) and - ( - prefixMsg = "starting with '" + escape(PrefixConstruction::prefix(s)) + "' and " and - not PrefixConstruction::prefix(s) = "" - or - PrefixConstruction::prefix(s) = "" and prefixMsg = "" - or - not exists(PrefixConstruction::prefix(s)) and prefixMsg = "" - ) -} diff --git a/java/ql/lib/semmle/code/java/security/performance/SuperlinearBackTracking.qll b/java/ql/lib/semmle/code/java/security/performance/SuperlinearBackTracking.qll index 4ba9520cdcc..05edbfd9777 100644 --- a/java/ql/lib/semmle/code/java/security/performance/SuperlinearBackTracking.qll +++ b/java/ql/lib/semmle/code/java/security/performance/SuperlinearBackTracking.qll @@ -391,7 +391,7 @@ predicate isPumpable(State pivot, State succ, string pump) { */ predicate polynimalReDoS(RegExpTerm t, string pump, string prefixMsg, RegExpTerm prev) { exists(State s, State pivot | - hasReDoSResult(t, pump, s, prefixMsg) and + ReDoSPruning::hasReDoSResult(t, pump, s, prefixMsg) and isPumpable(pivot, s, _) and prev = pivot.getRepr() ) diff --git a/java/ql/test/query-tests/security/CWE-730/ReDoS.ql b/java/ql/test/query-tests/security/CWE-730/ReDoS.ql index 79cb8243cd7..ae85707fc18 100644 --- a/java/ql/test/query-tests/security/CWE-730/ReDoS.ql +++ b/java/ql/test/query-tests/security/CWE-730/ReDoS.ql @@ -11,7 +11,7 @@ class HasExpRedos extends InlineExpectationsTest { override predicate hasActualResult(Location location, string element, string tag, string value) { tag = "hasExpRedos" and exists(RegExpTerm t, string pump, State s, string prefixMsg | - hasReDoSResult(t, pump, s, prefixMsg) and + ReDoSPruning::hasReDoSResult(t, pump, s, prefixMsg) and not t.getRegex().getAMode() = "VERBOSE" and value = "" and location = t.getLocation() and diff --git a/javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtil.qll b/javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtil.qll index 8aa348bf62f..8315d2b8225 100644 --- a/javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtil.qll +++ b/javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtil.qll @@ -14,45 +14,6 @@ import ReDoSUtilSpecific -/** - * A configuration for which parts of a regular expression should be considered relevant for - * the different predicates in `ReDoS.qll`. - * Used to adjust the computations for either superlinear or exponential backtracking. - */ -abstract class ReDoSConfiguration extends string { - bindingset[this] - ReDoSConfiguration() { any() } - - /** - * Holds if `state` with the pump string `pump` is a candidate for a - * ReDoS vulnerable state. - * This is used to determine which states are considered for the prefix/suffix construction. - */ - abstract predicate isReDoSCandidate(State state, string pump); -} - -/** - * Holds if repeating `pump` starting at `state` is a candidate for causing backtracking. - * No check whether a rejected suffix exists has been made. - */ -private predicate isReDoSCandidate(State state, string pump) { - any(ReDoSConfiguration conf).isReDoSCandidate(state, pump) and - ( - not any(ReDoSConfiguration conf).isReDoSCandidate(epsilonSucc+(state), _) - or - epsilonSucc+(state) = state and - state = - max(State s, Location l | - s = epsilonSucc+(state) and - l = s.getRepr().getLocation() and - any(ReDoSConfiguration conf).isReDoSCandidate(s, _) and - s.getRepr() instanceof InfiniteRepetitionQuantifier - | - s order by l.getStartLine(), l.getStartColumn(), l.getEndColumn(), l.getEndLine() - ) - ) -} - /** * Gets the char after `c` (from a simplified ASCII table). */ @@ -877,266 +838,341 @@ predicate isStartState(State state) { } /** - * Predicates for constructing a prefix string that leads to a given state. + * A configuration for which parts of a regular expression should be considered relevant for + * the different predicates in `ReDoS.qll`. + * Used to adjust the computations for either superlinear or exponential backtracking. */ -private module PrefixConstruction { - /** - * Holds if `state` is the textually last start state for the regular expression. - */ - private predicate lastStartState(State state) { - exists(RegExpRoot root | - state = - max(StateInPumpableRegexp s, Location l | - isStartState(s) and getRoot(s.getRepr()) = root and l = s.getRepr().getLocation() - | - s - order by - l.getStartLine(), l.getStartColumn(), s.getRepr().toString(), l.getEndColumn(), - l.getEndLine() - ) - ) - } +abstract class ReDoSConfiguration extends string { + bindingset[this] + ReDoSConfiguration() { any() } /** - * Holds if there exists any transition (Epsilon() or other) from `a` to `b`. + * Holds if `state` with the pump string `pump` is a candidate for a + * ReDoS vulnerable state. + * This is used to determine which states are considered for the prefix/suffix construction. */ - private predicate existsTransition(State a, State b) { delta(a, _, b) } - - /** - * Gets the minimum number of transitions it takes to reach `state` from the `start` state. - */ - int prefixLength(State start, State state) = - shortestDistances(lastStartState/1, existsTransition/2)(start, state, result) - - /** - * Gets the minimum number of transitions it takes to reach `state` from the start state. - */ - private int lengthFromStart(State state) { result = prefixLength(_, state) } - - /** - * Gets a string for which the regular expression will reach `state`. - * - * Has at most one result for any given `state`. - * This predicate will not always have a result even if there is a ReDoS issue in - * the regular expression. - */ - string prefix(State state) { - lastStartState(state) and - result = "" - or - // the search stops past the last redos candidate state. - lengthFromStart(state) <= max(lengthFromStart(any(State s | isReDoSCandidate(s, _)))) and - exists(State prev | - // select a unique predecessor (by an arbitrary measure) - prev = - min(State s, Location loc | - lengthFromStart(s) = lengthFromStart(state) - 1 and - loc = s.getRepr().getLocation() and - delta(s, _, state) - | - s - order by - loc.getStartLine(), loc.getStartColumn(), loc.getEndLine(), loc.getEndColumn(), - s.getRepr().toString() - ) - | - // greedy search for the shortest prefix - result = prefix(prev) and delta(prev, Epsilon(), state) - or - not delta(prev, Epsilon(), state) and - result = prefix(prev) + getCanonicalEdgeChar(prev, state) - ) - } - - /** - * Gets a canonical char for which there exists a transition from `prev` to `next` in the NFA. - */ - private string getCanonicalEdgeChar(State prev, State next) { - result = - min(string c | delta(prev, any(InputSymbol symbol | c = intersect(Any(), symbol)), next)) - } - - /** - * A state within a regular expression that has a pumpable state. - */ - class StateInPumpableRegexp extends State { - pragma[noinline] - StateInPumpableRegexp() { - exists(State s | isReDoSCandidate(s, _) | getRoot(s.getRepr()) = getRoot(this.getRepr())) - } - } + abstract predicate isReDoSCandidate(State state, string pump); } -/** - * Predicates for testing the presence of a rejecting suffix. - * - * These predicates are used to ensure that the all states reached from the fork - * by repeating `w` have a rejecting suffix. - * - * For example, a regexp like `/^(a+)+/` will accept any string as long the prefix is - * some number of `"a"`s, and it is therefore not possible to construct a rejecting suffix. - * - * A regexp like `/(a+)+$/` or `/(a+)+b/` trivially has a rejecting suffix, - * as the suffix "X" will cause both the regular expressions to be rejected. - * - * The string `w` is repeated any number of times because it needs to be - * infinitely repeatedable for the attack to work. - * For the regular expression `/((ab)+)*abab/` the accepting state is not reachable from the fork - * using epsilon transitions. But any attempt at repeating `w` will end in a state that accepts all suffixes. - */ -private module SuffixConstruction { - import PrefixConstruction - +module ReDoSPruning { /** - * Holds if all states reachable from `fork` by repeating `w` - * are likely rejectable by appending some suffix. + * Holds if repeating `pump` starting at `state` is a candidate for causing backtracking. + * No check whether a rejected suffix exists has been made. */ - predicate reachesOnlyRejectableSuffixes(State fork, string w) { - isReDoSCandidate(fork, w) and - forex(State next | next = process(fork, w, w.length() - 1) | isLikelyRejectable(next)) - } - - /** - * Holds if there likely exists a suffix starting from `s` that leads to the regular expression being rejected. - * This predicate might find impossible suffixes when searching for suffixes of length > 1, which can cause FPs. - */ - pragma[noinline] - private predicate isLikelyRejectable(StateInPumpableRegexp s) { - // exists a reject edge with some char. - hasRejectEdge(s) - or - hasEdgeToLikelyRejectable(s) - or - // stopping here is rejection - isRejectState(s) - } - - /** - * Holds if `s` is not an accept state, and there is no epsilon transition to an accept state. - */ - predicate isRejectState(StateInPumpableRegexp s) { not epsilonSucc*(s) = Accept(_) } - - /** - * Holds if there is likely a non-empty suffix leading to rejection starting in `s`. - */ - pragma[noopt] - predicate hasEdgeToLikelyRejectable(StateInPumpableRegexp s) { - // all edges (at least one) with some char leads to another state that is rejectable. - // the `next` states might not share a common suffix, which can cause FPs. - exists(string char | char = hasEdgeToLikelyRejectableHelper(s) | - // noopt to force `hasEdgeToLikelyRejectableHelper` to be first in the join-order. - exists(State next | deltaClosedChar(s, char, next) | isLikelyRejectable(next)) and - forall(State next | deltaClosedChar(s, char, next) | isLikelyRejectable(next)) - ) - } - - /** - * Gets a char for there exists a transition away from `s`, - * and `s` has not been found to be rejectable by `hasRejectEdge` or `isRejectState`. - */ - pragma[noinline] - private string hasEdgeToLikelyRejectableHelper(StateInPumpableRegexp s) { - not hasRejectEdge(s) and - not isRejectState(s) and - deltaClosedChar(s, result, _) - } - - /** - * Holds if there is a state `next` that can be reached from `prev` - * along epsilon edges, such that there is a transition from - * `prev` to `next` that the character symbol `char`. - */ - predicate deltaClosedChar(StateInPumpableRegexp prev, string char, StateInPumpableRegexp next) { - deltaClosed(prev, getAnInputSymbolMatchingRelevant(char), next) - } - - pragma[noinline] - InputSymbol getAnInputSymbolMatchingRelevant(string char) { - char = relevant(_) and - result = getAnInputSymbolMatching(char) - } - - /** - * Gets a char used for finding possible suffixes inside `root`. - */ - pragma[noinline] - private string relevant(RegExpRoot root) { - exists(ascii(result)) and exists(root) - or - exists(InputSymbol s | belongsTo(s, root) | result = intersect(s, _)) - or - // The characters from `hasSimpleRejectEdge`. Only `\n` is really needed (as `\n` is not in the `ascii` relation). - // The three chars must be kept in sync with `hasSimpleRejectEdge`. - result = ["|", "\n", "Z"] and exists(root) - } - - /** - * Holds if there exists a `char` such that there is no edge from `s` labeled `char` in our NFA. - * The NFA does not model reject states, so the above is the same as saying there is a reject edge. - */ - private predicate hasRejectEdge(State s) { - hasSimpleRejectEdge(s) - or - not hasSimpleRejectEdge(s) and - exists(string char | char = relevant(getRoot(s.getRepr())) | not deltaClosedChar(s, char, _)) - } - - /** - * Holds if there is no edge from `s` labeled with "|", "\n", or "Z" in our NFA. - * This predicate is used as a cheap pre-processing to speed up `hasRejectEdge`. - */ - private predicate hasSimpleRejectEdge(State s) { - // The three chars were chosen arbitrarily. The three chars must be kept in sync with `relevant`. - exists(string char | char = ["|", "\n", "Z"] | not deltaClosedChar(s, char, _)) - } - - /** - * Gets a state that can be reached from pumpable `fork` consuming all - * chars in `w` any number of times followed by the first `i+1` characters of `w`. - */ - pragma[noopt] - private State process(State fork, string w, int i) { - exists(State prev | prev = getProcessPrevious(fork, i, w) | - exists(string char, InputSymbol sym | - char = w.charAt(i) and - deltaClosed(prev, sym, result) and - // noopt to prevent joining `prev` with all possible `chars` that could transition away from `prev`. - // Instead only join with the set of `chars` where a relevant `InputSymbol` has already been found. - sym = getAProcessInputSymbol(char) - ) - ) - } - - /** - * Gets a state that can be reached from pumpable `fork` consuming all - * chars in `w` any number of times followed by the first `i` characters of `w`. - */ - private State getProcessPrevious(State fork, int i, string w) { - isReDoSCandidate(fork, w) and + private predicate isReDoSCandidate(State state, string pump) { + any(ReDoSConfiguration conf).isReDoSCandidate(state, pump) and ( - i = 0 and result = fork + not any(ReDoSConfiguration conf).isReDoSCandidate(epsilonSucc+(state), _) or - result = process(fork, w, i - 1) - or - // repeat until fixpoint - i = 0 and - result = process(fork, w, w.length() - 1) + epsilonSucc+(state) = state and + state = + max(State s, Location l | + s = epsilonSucc+(state) and + l = s.getRepr().getLocation() and + any(ReDoSConfiguration conf).isReDoSCandidate(s, _) and + s.getRepr() instanceof InfiniteRepetitionQuantifier + | + s order by l.getStartLine(), l.getStartColumn(), l.getEndColumn(), l.getEndLine() + ) ) } /** - * Gets an InputSymbol that matches `char`. - * The predicate is specialized to only have a result for the `char`s that are relevant for the `process` predicate. + * Predicates for constructing a prefix string that leads to a given state. */ - private InputSymbol getAProcessInputSymbol(string char) { - char = getAProcessChar() and - result = getAnInputSymbolMatching(char) + private module PrefixConstruction { + /** + * Holds if `state` is the textually last start state for the regular expression. + */ + private predicate lastStartState(State state) { + exists(RegExpRoot root | + state = + max(StateInPumpableRegexp s, Location l | + isStartState(s) and getRoot(s.getRepr()) = root and l = s.getRepr().getLocation() + | + s + order by + l.getStartLine(), l.getStartColumn(), s.getRepr().toString(), l.getEndColumn(), + l.getEndLine() + ) + ) + } + + /** + * Holds if there exists any transition (Epsilon() or other) from `a` to `b`. + */ + private predicate existsTransition(State a, State b) { delta(a, _, b) } + + /** + * Gets the minimum number of transitions it takes to reach `state` from the `start` state. + */ + int prefixLength(State start, State state) = + shortestDistances(lastStartState/1, existsTransition/2)(start, state, result) + + /** + * Gets the minimum number of transitions it takes to reach `state` from the start state. + */ + private int lengthFromStart(State state) { result = prefixLength(_, state) } + + /** + * Gets a string for which the regular expression will reach `state`. + * + * Has at most one result for any given `state`. + * This predicate will not always have a result even if there is a ReDoS issue in + * the regular expression. + */ + string prefix(State state) { + lastStartState(state) and + result = "" + or + // the search stops past the last redos candidate state. + lengthFromStart(state) <= max(lengthFromStart(any(State s | isReDoSCandidate(s, _)))) and + exists(State prev | + // select a unique predecessor (by an arbitrary measure) + prev = + min(State s, Location loc | + lengthFromStart(s) = lengthFromStart(state) - 1 and + loc = s.getRepr().getLocation() and + delta(s, _, state) + | + s + order by + loc.getStartLine(), loc.getStartColumn(), loc.getEndLine(), loc.getEndColumn(), + s.getRepr().toString() + ) + | + // greedy search for the shortest prefix + result = prefix(prev) and delta(prev, Epsilon(), state) + or + not delta(prev, Epsilon(), state) and + result = prefix(prev) + getCanonicalEdgeChar(prev, state) + ) + } + + /** + * Gets a canonical char for which there exists a transition from `prev` to `next` in the NFA. + */ + private string getCanonicalEdgeChar(State prev, State next) { + result = + min(string c | delta(prev, any(InputSymbol symbol | c = intersect(Any(), symbol)), next)) + } + + /** + * A state within a regular expression that has a pumpable state. + */ + class StateInPumpableRegexp extends State { + pragma[noinline] + StateInPumpableRegexp() { + exists(State s | isReDoSCandidate(s, _) | getRoot(s.getRepr()) = getRoot(this.getRepr())) + } + } } /** - * Gets a `char` that occurs in a `pump` string. + * Predicates for testing the presence of a rejecting suffix. + * + * These predicates are used to ensure that the all states reached from the fork + * by repeating `w` have a rejecting suffix. + * + * For example, a regexp like `/^(a+)+/` will accept any string as long the prefix is + * some number of `"a"`s, and it is therefore not possible to construct a rejecting suffix. + * + * A regexp like `/(a+)+$/` or `/(a+)+b/` trivially has a rejecting suffix, + * as the suffix "X" will cause both the regular expressions to be rejected. + * + * The string `w` is repeated any number of times because it needs to be + * infinitely repeatedable for the attack to work. + * For the regular expression `/((ab)+)*abab/` the accepting state is not reachable from the fork + * using epsilon transitions. But any attempt at repeating `w` will end in a state that accepts all suffixes. */ - private string getAProcessChar() { result = any(string s | isReDoSCandidate(_, s)).charAt(_) } + private module SuffixConstruction { + import PrefixConstruction + + /** + * Holds if all states reachable from `fork` by repeating `w` + * are likely rejectable by appending some suffix. + */ + predicate reachesOnlyRejectableSuffixes(State fork, string w) { + isReDoSCandidate(fork, w) and + forex(State next | next = process(fork, w, w.length() - 1) | isLikelyRejectable(next)) + } + + /** + * Holds if there likely exists a suffix starting from `s` that leads to the regular expression being rejected. + * This predicate might find impossible suffixes when searching for suffixes of length > 1, which can cause FPs. + */ + pragma[noinline] + private predicate isLikelyRejectable(StateInPumpableRegexp s) { + // exists a reject edge with some char. + hasRejectEdge(s) + or + hasEdgeToLikelyRejectable(s) + or + // stopping here is rejection + isRejectState(s) + } + + /** + * Holds if `s` is not an accept state, and there is no epsilon transition to an accept state. + */ + predicate isRejectState(StateInPumpableRegexp s) { not epsilonSucc*(s) = Accept(_) } + + /** + * Holds if there is likely a non-empty suffix leading to rejection starting in `s`. + */ + pragma[noopt] + predicate hasEdgeToLikelyRejectable(StateInPumpableRegexp s) { + // all edges (at least one) with some char leads to another state that is rejectable. + // the `next` states might not share a common suffix, which can cause FPs. + exists(string char | char = hasEdgeToLikelyRejectableHelper(s) | + // noopt to force `hasEdgeToLikelyRejectableHelper` to be first in the join-order. + exists(State next | deltaClosedChar(s, char, next) | isLikelyRejectable(next)) and + forall(State next | deltaClosedChar(s, char, next) | isLikelyRejectable(next)) + ) + } + + /** + * Gets a char for there exists a transition away from `s`, + * and `s` has not been found to be rejectable by `hasRejectEdge` or `isRejectState`. + */ + pragma[noinline] + private string hasEdgeToLikelyRejectableHelper(StateInPumpableRegexp s) { + not hasRejectEdge(s) and + not isRejectState(s) and + deltaClosedChar(s, result, _) + } + + /** + * Holds if there is a state `next` that can be reached from `prev` + * along epsilon edges, such that there is a transition from + * `prev` to `next` that the character symbol `char`. + */ + predicate deltaClosedChar(StateInPumpableRegexp prev, string char, StateInPumpableRegexp next) { + deltaClosed(prev, getAnInputSymbolMatchingRelevant(char), next) + } + + pragma[noinline] + InputSymbol getAnInputSymbolMatchingRelevant(string char) { + char = relevant(_) and + result = getAnInputSymbolMatching(char) + } + + /** + * Gets a char used for finding possible suffixes inside `root`. + */ + pragma[noinline] + private string relevant(RegExpRoot root) { + exists(ascii(result)) and exists(root) + or + exists(InputSymbol s | belongsTo(s, root) | result = intersect(s, _)) + or + // The characters from `hasSimpleRejectEdge`. Only `\n` is really needed (as `\n` is not in the `ascii` relation). + // The three chars must be kept in sync with `hasSimpleRejectEdge`. + result = ["|", "\n", "Z"] and exists(root) + } + + /** + * Holds if there exists a `char` such that there is no edge from `s` labeled `char` in our NFA. + * The NFA does not model reject states, so the above is the same as saying there is a reject edge. + */ + private predicate hasRejectEdge(State s) { + hasSimpleRejectEdge(s) + or + not hasSimpleRejectEdge(s) and + exists(string char | char = relevant(getRoot(s.getRepr())) | not deltaClosedChar(s, char, _)) + } + + /** + * Holds if there is no edge from `s` labeled with "|", "\n", or "Z" in our NFA. + * This predicate is used as a cheap pre-processing to speed up `hasRejectEdge`. + */ + private predicate hasSimpleRejectEdge(State s) { + // The three chars were chosen arbitrarily. The three chars must be kept in sync with `relevant`. + exists(string char | char = ["|", "\n", "Z"] | not deltaClosedChar(s, char, _)) + } + + /** + * Gets a state that can be reached from pumpable `fork` consuming all + * chars in `w` any number of times followed by the first `i+1` characters of `w`. + */ + pragma[noopt] + private State process(State fork, string w, int i) { + exists(State prev | prev = getProcessPrevious(fork, i, w) | + exists(string char, InputSymbol sym | + char = w.charAt(i) and + deltaClosed(prev, sym, result) and + // noopt to prevent joining `prev` with all possible `chars` that could transition away from `prev`. + // Instead only join with the set of `chars` where a relevant `InputSymbol` has already been found. + sym = getAProcessInputSymbol(char) + ) + ) + } + + /** + * Gets a state that can be reached from pumpable `fork` consuming all + * chars in `w` any number of times followed by the first `i` characters of `w`. + */ + private State getProcessPrevious(State fork, int i, string w) { + isReDoSCandidate(fork, w) and + ( + i = 0 and result = fork + or + result = process(fork, w, i - 1) + or + // repeat until fixpoint + i = 0 and + result = process(fork, w, w.length() - 1) + ) + } + + /** + * Gets an InputSymbol that matches `char`. + * The predicate is specialized to only have a result for the `char`s that are relevant for the `process` predicate. + */ + private InputSymbol getAProcessInputSymbol(string char) { + char = getAProcessChar() and + result = getAnInputSymbolMatching(char) + } + + /** + * Gets a `char` that occurs in a `pump` string. + */ + private string getAProcessChar() { result = any(string s | isReDoSCandidate(_, s)).charAt(_) } + } + + /** + * Holds if `term` may cause superlinear backtracking on strings containing many repetitions of `pump`. + * Gets the shortest string that causes superlinear backtracking. + */ + private predicate isReDoSAttackable(RegExpTerm term, string pump, State s) { + exists(int i, string c | s = Match(term, i) | + c = + min(string w | + any(ReDoSConfiguration conf).isReDoSCandidate(s, w) and + SuffixConstruction::reachesOnlyRejectableSuffixes(s, w) + | + w order by w.length(), w + ) and + pump = escape(rotate(c, i)) + ) + } + + /** + * Holds if the state `s` (represented by the term `t`) can have backtracking with repetitions of `pump`. + * + * `prefixMsg` contains a friendly message for a prefix that reaches `s` (or `prefixMsg` is the empty string if the prefix is empty or if no prefix could be found). + */ + predicate hasReDoSResult(RegExpTerm t, string pump, State s, string prefixMsg) { + isReDoSAttackable(t, pump, s) and + ( + prefixMsg = "starting with '" + escape(PrefixConstruction::prefix(s)) + "' and " and + not PrefixConstruction::prefix(s) = "" + or + PrefixConstruction::prefix(s) = "" and prefixMsg = "" + or + not exists(PrefixConstruction::prefix(s)) and prefixMsg = "" + ) + } } /** @@ -1162,37 +1198,3 @@ bindingset[str, i] private string rotate(string str, int i) { result = str.suffix(str.length() - i) + str.prefix(str.length() - i) } - -/** - * Holds if `term` may cause superlinear backtracking on strings containing many repetitions of `pump`. - * Gets the shortest string that causes superlinear backtracking. - */ -private predicate isReDoSAttackable(RegExpTerm term, string pump, State s) { - exists(int i, string c | s = Match(term, i) | - c = - min(string w | - any(ReDoSConfiguration conf).isReDoSCandidate(s, w) and - SuffixConstruction::reachesOnlyRejectableSuffixes(s, w) - | - w order by w.length(), w - ) and - pump = escape(rotate(c, i)) - ) -} - -/** - * Holds if the state `s` (represented by the term `t`) can have backtracking with repetitions of `pump`. - * - * `prefixMsg` contains a friendly message for a prefix that reaches `s` (or `prefixMsg` is the empty string if the prefix is empty or if no prefix could be found). - */ -predicate hasReDoSResult(RegExpTerm t, string pump, State s, string prefixMsg) { - isReDoSAttackable(t, pump, s) and - ( - prefixMsg = "starting with '" + escape(PrefixConstruction::prefix(s)) + "' and " and - not PrefixConstruction::prefix(s) = "" - or - PrefixConstruction::prefix(s) = "" and prefixMsg = "" - or - not exists(PrefixConstruction::prefix(s)) and prefixMsg = "" - ) -} diff --git a/javascript/ql/lib/semmle/javascript/security/performance/SuperlinearBackTracking.qll b/javascript/ql/lib/semmle/javascript/security/performance/SuperlinearBackTracking.qll index 4ba9520cdcc..05edbfd9777 100644 --- a/javascript/ql/lib/semmle/javascript/security/performance/SuperlinearBackTracking.qll +++ b/javascript/ql/lib/semmle/javascript/security/performance/SuperlinearBackTracking.qll @@ -391,7 +391,7 @@ predicate isPumpable(State pivot, State succ, string pump) { */ predicate polynimalReDoS(RegExpTerm t, string pump, string prefixMsg, RegExpTerm prev) { exists(State s, State pivot | - hasReDoSResult(t, pump, s, prefixMsg) and + ReDoSPruning::hasReDoSResult(t, pump, s, prefixMsg) and isPumpable(pivot, s, _) and prev = pivot.getRepr() ) diff --git a/javascript/ql/src/Performance/ReDoS.ql b/javascript/ql/src/Performance/ReDoS.ql index bda773bf0e7..854f3653e8d 100644 --- a/javascript/ql/src/Performance/ReDoS.ql +++ b/javascript/ql/src/Performance/ReDoS.ql @@ -19,7 +19,7 @@ import semmle.javascript.security.performance.ReDoSUtil import semmle.javascript.security.performance.ExponentialBackTracking from RegExpTerm t, string pump, State s, string prefixMsg -where hasReDoSResult(t, pump, s, prefixMsg) +where ReDoSPruning::hasReDoSResult(t, pump, s, prefixMsg) select t, "This part of the regular expression may cause exponential backtracking on strings " + prefixMsg + "containing many repetitions of '" + pump + "'." diff --git a/python/ql/lib/semmle/python/security/performance/ReDoSUtil.qll b/python/ql/lib/semmle/python/security/performance/ReDoSUtil.qll index 8aa348bf62f..8315d2b8225 100644 --- a/python/ql/lib/semmle/python/security/performance/ReDoSUtil.qll +++ b/python/ql/lib/semmle/python/security/performance/ReDoSUtil.qll @@ -14,45 +14,6 @@ import ReDoSUtilSpecific -/** - * A configuration for which parts of a regular expression should be considered relevant for - * the different predicates in `ReDoS.qll`. - * Used to adjust the computations for either superlinear or exponential backtracking. - */ -abstract class ReDoSConfiguration extends string { - bindingset[this] - ReDoSConfiguration() { any() } - - /** - * Holds if `state` with the pump string `pump` is a candidate for a - * ReDoS vulnerable state. - * This is used to determine which states are considered for the prefix/suffix construction. - */ - abstract predicate isReDoSCandidate(State state, string pump); -} - -/** - * Holds if repeating `pump` starting at `state` is a candidate for causing backtracking. - * No check whether a rejected suffix exists has been made. - */ -private predicate isReDoSCandidate(State state, string pump) { - any(ReDoSConfiguration conf).isReDoSCandidate(state, pump) and - ( - not any(ReDoSConfiguration conf).isReDoSCandidate(epsilonSucc+(state), _) - or - epsilonSucc+(state) = state and - state = - max(State s, Location l | - s = epsilonSucc+(state) and - l = s.getRepr().getLocation() and - any(ReDoSConfiguration conf).isReDoSCandidate(s, _) and - s.getRepr() instanceof InfiniteRepetitionQuantifier - | - s order by l.getStartLine(), l.getStartColumn(), l.getEndColumn(), l.getEndLine() - ) - ) -} - /** * Gets the char after `c` (from a simplified ASCII table). */ @@ -877,266 +838,341 @@ predicate isStartState(State state) { } /** - * Predicates for constructing a prefix string that leads to a given state. + * A configuration for which parts of a regular expression should be considered relevant for + * the different predicates in `ReDoS.qll`. + * Used to adjust the computations for either superlinear or exponential backtracking. */ -private module PrefixConstruction { - /** - * Holds if `state` is the textually last start state for the regular expression. - */ - private predicate lastStartState(State state) { - exists(RegExpRoot root | - state = - max(StateInPumpableRegexp s, Location l | - isStartState(s) and getRoot(s.getRepr()) = root and l = s.getRepr().getLocation() - | - s - order by - l.getStartLine(), l.getStartColumn(), s.getRepr().toString(), l.getEndColumn(), - l.getEndLine() - ) - ) - } +abstract class ReDoSConfiguration extends string { + bindingset[this] + ReDoSConfiguration() { any() } /** - * Holds if there exists any transition (Epsilon() or other) from `a` to `b`. + * Holds if `state` with the pump string `pump` is a candidate for a + * ReDoS vulnerable state. + * This is used to determine which states are considered for the prefix/suffix construction. */ - private predicate existsTransition(State a, State b) { delta(a, _, b) } - - /** - * Gets the minimum number of transitions it takes to reach `state` from the `start` state. - */ - int prefixLength(State start, State state) = - shortestDistances(lastStartState/1, existsTransition/2)(start, state, result) - - /** - * Gets the minimum number of transitions it takes to reach `state` from the start state. - */ - private int lengthFromStart(State state) { result = prefixLength(_, state) } - - /** - * Gets a string for which the regular expression will reach `state`. - * - * Has at most one result for any given `state`. - * This predicate will not always have a result even if there is a ReDoS issue in - * the regular expression. - */ - string prefix(State state) { - lastStartState(state) and - result = "" - or - // the search stops past the last redos candidate state. - lengthFromStart(state) <= max(lengthFromStart(any(State s | isReDoSCandidate(s, _)))) and - exists(State prev | - // select a unique predecessor (by an arbitrary measure) - prev = - min(State s, Location loc | - lengthFromStart(s) = lengthFromStart(state) - 1 and - loc = s.getRepr().getLocation() and - delta(s, _, state) - | - s - order by - loc.getStartLine(), loc.getStartColumn(), loc.getEndLine(), loc.getEndColumn(), - s.getRepr().toString() - ) - | - // greedy search for the shortest prefix - result = prefix(prev) and delta(prev, Epsilon(), state) - or - not delta(prev, Epsilon(), state) and - result = prefix(prev) + getCanonicalEdgeChar(prev, state) - ) - } - - /** - * Gets a canonical char for which there exists a transition from `prev` to `next` in the NFA. - */ - private string getCanonicalEdgeChar(State prev, State next) { - result = - min(string c | delta(prev, any(InputSymbol symbol | c = intersect(Any(), symbol)), next)) - } - - /** - * A state within a regular expression that has a pumpable state. - */ - class StateInPumpableRegexp extends State { - pragma[noinline] - StateInPumpableRegexp() { - exists(State s | isReDoSCandidate(s, _) | getRoot(s.getRepr()) = getRoot(this.getRepr())) - } - } + abstract predicate isReDoSCandidate(State state, string pump); } -/** - * Predicates for testing the presence of a rejecting suffix. - * - * These predicates are used to ensure that the all states reached from the fork - * by repeating `w` have a rejecting suffix. - * - * For example, a regexp like `/^(a+)+/` will accept any string as long the prefix is - * some number of `"a"`s, and it is therefore not possible to construct a rejecting suffix. - * - * A regexp like `/(a+)+$/` or `/(a+)+b/` trivially has a rejecting suffix, - * as the suffix "X" will cause both the regular expressions to be rejected. - * - * The string `w` is repeated any number of times because it needs to be - * infinitely repeatedable for the attack to work. - * For the regular expression `/((ab)+)*abab/` the accepting state is not reachable from the fork - * using epsilon transitions. But any attempt at repeating `w` will end in a state that accepts all suffixes. - */ -private module SuffixConstruction { - import PrefixConstruction - +module ReDoSPruning { /** - * Holds if all states reachable from `fork` by repeating `w` - * are likely rejectable by appending some suffix. + * Holds if repeating `pump` starting at `state` is a candidate for causing backtracking. + * No check whether a rejected suffix exists has been made. */ - predicate reachesOnlyRejectableSuffixes(State fork, string w) { - isReDoSCandidate(fork, w) and - forex(State next | next = process(fork, w, w.length() - 1) | isLikelyRejectable(next)) - } - - /** - * Holds if there likely exists a suffix starting from `s` that leads to the regular expression being rejected. - * This predicate might find impossible suffixes when searching for suffixes of length > 1, which can cause FPs. - */ - pragma[noinline] - private predicate isLikelyRejectable(StateInPumpableRegexp s) { - // exists a reject edge with some char. - hasRejectEdge(s) - or - hasEdgeToLikelyRejectable(s) - or - // stopping here is rejection - isRejectState(s) - } - - /** - * Holds if `s` is not an accept state, and there is no epsilon transition to an accept state. - */ - predicate isRejectState(StateInPumpableRegexp s) { not epsilonSucc*(s) = Accept(_) } - - /** - * Holds if there is likely a non-empty suffix leading to rejection starting in `s`. - */ - pragma[noopt] - predicate hasEdgeToLikelyRejectable(StateInPumpableRegexp s) { - // all edges (at least one) with some char leads to another state that is rejectable. - // the `next` states might not share a common suffix, which can cause FPs. - exists(string char | char = hasEdgeToLikelyRejectableHelper(s) | - // noopt to force `hasEdgeToLikelyRejectableHelper` to be first in the join-order. - exists(State next | deltaClosedChar(s, char, next) | isLikelyRejectable(next)) and - forall(State next | deltaClosedChar(s, char, next) | isLikelyRejectable(next)) - ) - } - - /** - * Gets a char for there exists a transition away from `s`, - * and `s` has not been found to be rejectable by `hasRejectEdge` or `isRejectState`. - */ - pragma[noinline] - private string hasEdgeToLikelyRejectableHelper(StateInPumpableRegexp s) { - not hasRejectEdge(s) and - not isRejectState(s) and - deltaClosedChar(s, result, _) - } - - /** - * Holds if there is a state `next` that can be reached from `prev` - * along epsilon edges, such that there is a transition from - * `prev` to `next` that the character symbol `char`. - */ - predicate deltaClosedChar(StateInPumpableRegexp prev, string char, StateInPumpableRegexp next) { - deltaClosed(prev, getAnInputSymbolMatchingRelevant(char), next) - } - - pragma[noinline] - InputSymbol getAnInputSymbolMatchingRelevant(string char) { - char = relevant(_) and - result = getAnInputSymbolMatching(char) - } - - /** - * Gets a char used for finding possible suffixes inside `root`. - */ - pragma[noinline] - private string relevant(RegExpRoot root) { - exists(ascii(result)) and exists(root) - or - exists(InputSymbol s | belongsTo(s, root) | result = intersect(s, _)) - or - // The characters from `hasSimpleRejectEdge`. Only `\n` is really needed (as `\n` is not in the `ascii` relation). - // The three chars must be kept in sync with `hasSimpleRejectEdge`. - result = ["|", "\n", "Z"] and exists(root) - } - - /** - * Holds if there exists a `char` such that there is no edge from `s` labeled `char` in our NFA. - * The NFA does not model reject states, so the above is the same as saying there is a reject edge. - */ - private predicate hasRejectEdge(State s) { - hasSimpleRejectEdge(s) - or - not hasSimpleRejectEdge(s) and - exists(string char | char = relevant(getRoot(s.getRepr())) | not deltaClosedChar(s, char, _)) - } - - /** - * Holds if there is no edge from `s` labeled with "|", "\n", or "Z" in our NFA. - * This predicate is used as a cheap pre-processing to speed up `hasRejectEdge`. - */ - private predicate hasSimpleRejectEdge(State s) { - // The three chars were chosen arbitrarily. The three chars must be kept in sync with `relevant`. - exists(string char | char = ["|", "\n", "Z"] | not deltaClosedChar(s, char, _)) - } - - /** - * Gets a state that can be reached from pumpable `fork` consuming all - * chars in `w` any number of times followed by the first `i+1` characters of `w`. - */ - pragma[noopt] - private State process(State fork, string w, int i) { - exists(State prev | prev = getProcessPrevious(fork, i, w) | - exists(string char, InputSymbol sym | - char = w.charAt(i) and - deltaClosed(prev, sym, result) and - // noopt to prevent joining `prev` with all possible `chars` that could transition away from `prev`. - // Instead only join with the set of `chars` where a relevant `InputSymbol` has already been found. - sym = getAProcessInputSymbol(char) - ) - ) - } - - /** - * Gets a state that can be reached from pumpable `fork` consuming all - * chars in `w` any number of times followed by the first `i` characters of `w`. - */ - private State getProcessPrevious(State fork, int i, string w) { - isReDoSCandidate(fork, w) and + private predicate isReDoSCandidate(State state, string pump) { + any(ReDoSConfiguration conf).isReDoSCandidate(state, pump) and ( - i = 0 and result = fork + not any(ReDoSConfiguration conf).isReDoSCandidate(epsilonSucc+(state), _) or - result = process(fork, w, i - 1) - or - // repeat until fixpoint - i = 0 and - result = process(fork, w, w.length() - 1) + epsilonSucc+(state) = state and + state = + max(State s, Location l | + s = epsilonSucc+(state) and + l = s.getRepr().getLocation() and + any(ReDoSConfiguration conf).isReDoSCandidate(s, _) and + s.getRepr() instanceof InfiniteRepetitionQuantifier + | + s order by l.getStartLine(), l.getStartColumn(), l.getEndColumn(), l.getEndLine() + ) ) } /** - * Gets an InputSymbol that matches `char`. - * The predicate is specialized to only have a result for the `char`s that are relevant for the `process` predicate. + * Predicates for constructing a prefix string that leads to a given state. */ - private InputSymbol getAProcessInputSymbol(string char) { - char = getAProcessChar() and - result = getAnInputSymbolMatching(char) + private module PrefixConstruction { + /** + * Holds if `state` is the textually last start state for the regular expression. + */ + private predicate lastStartState(State state) { + exists(RegExpRoot root | + state = + max(StateInPumpableRegexp s, Location l | + isStartState(s) and getRoot(s.getRepr()) = root and l = s.getRepr().getLocation() + | + s + order by + l.getStartLine(), l.getStartColumn(), s.getRepr().toString(), l.getEndColumn(), + l.getEndLine() + ) + ) + } + + /** + * Holds if there exists any transition (Epsilon() or other) from `a` to `b`. + */ + private predicate existsTransition(State a, State b) { delta(a, _, b) } + + /** + * Gets the minimum number of transitions it takes to reach `state` from the `start` state. + */ + int prefixLength(State start, State state) = + shortestDistances(lastStartState/1, existsTransition/2)(start, state, result) + + /** + * Gets the minimum number of transitions it takes to reach `state` from the start state. + */ + private int lengthFromStart(State state) { result = prefixLength(_, state) } + + /** + * Gets a string for which the regular expression will reach `state`. + * + * Has at most one result for any given `state`. + * This predicate will not always have a result even if there is a ReDoS issue in + * the regular expression. + */ + string prefix(State state) { + lastStartState(state) and + result = "" + or + // the search stops past the last redos candidate state. + lengthFromStart(state) <= max(lengthFromStart(any(State s | isReDoSCandidate(s, _)))) and + exists(State prev | + // select a unique predecessor (by an arbitrary measure) + prev = + min(State s, Location loc | + lengthFromStart(s) = lengthFromStart(state) - 1 and + loc = s.getRepr().getLocation() and + delta(s, _, state) + | + s + order by + loc.getStartLine(), loc.getStartColumn(), loc.getEndLine(), loc.getEndColumn(), + s.getRepr().toString() + ) + | + // greedy search for the shortest prefix + result = prefix(prev) and delta(prev, Epsilon(), state) + or + not delta(prev, Epsilon(), state) and + result = prefix(prev) + getCanonicalEdgeChar(prev, state) + ) + } + + /** + * Gets a canonical char for which there exists a transition from `prev` to `next` in the NFA. + */ + private string getCanonicalEdgeChar(State prev, State next) { + result = + min(string c | delta(prev, any(InputSymbol symbol | c = intersect(Any(), symbol)), next)) + } + + /** + * A state within a regular expression that has a pumpable state. + */ + class StateInPumpableRegexp extends State { + pragma[noinline] + StateInPumpableRegexp() { + exists(State s | isReDoSCandidate(s, _) | getRoot(s.getRepr()) = getRoot(this.getRepr())) + } + } } /** - * Gets a `char` that occurs in a `pump` string. + * Predicates for testing the presence of a rejecting suffix. + * + * These predicates are used to ensure that the all states reached from the fork + * by repeating `w` have a rejecting suffix. + * + * For example, a regexp like `/^(a+)+/` will accept any string as long the prefix is + * some number of `"a"`s, and it is therefore not possible to construct a rejecting suffix. + * + * A regexp like `/(a+)+$/` or `/(a+)+b/` trivially has a rejecting suffix, + * as the suffix "X" will cause both the regular expressions to be rejected. + * + * The string `w` is repeated any number of times because it needs to be + * infinitely repeatedable for the attack to work. + * For the regular expression `/((ab)+)*abab/` the accepting state is not reachable from the fork + * using epsilon transitions. But any attempt at repeating `w` will end in a state that accepts all suffixes. */ - private string getAProcessChar() { result = any(string s | isReDoSCandidate(_, s)).charAt(_) } + private module SuffixConstruction { + import PrefixConstruction + + /** + * Holds if all states reachable from `fork` by repeating `w` + * are likely rejectable by appending some suffix. + */ + predicate reachesOnlyRejectableSuffixes(State fork, string w) { + isReDoSCandidate(fork, w) and + forex(State next | next = process(fork, w, w.length() - 1) | isLikelyRejectable(next)) + } + + /** + * Holds if there likely exists a suffix starting from `s` that leads to the regular expression being rejected. + * This predicate might find impossible suffixes when searching for suffixes of length > 1, which can cause FPs. + */ + pragma[noinline] + private predicate isLikelyRejectable(StateInPumpableRegexp s) { + // exists a reject edge with some char. + hasRejectEdge(s) + or + hasEdgeToLikelyRejectable(s) + or + // stopping here is rejection + isRejectState(s) + } + + /** + * Holds if `s` is not an accept state, and there is no epsilon transition to an accept state. + */ + predicate isRejectState(StateInPumpableRegexp s) { not epsilonSucc*(s) = Accept(_) } + + /** + * Holds if there is likely a non-empty suffix leading to rejection starting in `s`. + */ + pragma[noopt] + predicate hasEdgeToLikelyRejectable(StateInPumpableRegexp s) { + // all edges (at least one) with some char leads to another state that is rejectable. + // the `next` states might not share a common suffix, which can cause FPs. + exists(string char | char = hasEdgeToLikelyRejectableHelper(s) | + // noopt to force `hasEdgeToLikelyRejectableHelper` to be first in the join-order. + exists(State next | deltaClosedChar(s, char, next) | isLikelyRejectable(next)) and + forall(State next | deltaClosedChar(s, char, next) | isLikelyRejectable(next)) + ) + } + + /** + * Gets a char for there exists a transition away from `s`, + * and `s` has not been found to be rejectable by `hasRejectEdge` or `isRejectState`. + */ + pragma[noinline] + private string hasEdgeToLikelyRejectableHelper(StateInPumpableRegexp s) { + not hasRejectEdge(s) and + not isRejectState(s) and + deltaClosedChar(s, result, _) + } + + /** + * Holds if there is a state `next` that can be reached from `prev` + * along epsilon edges, such that there is a transition from + * `prev` to `next` that the character symbol `char`. + */ + predicate deltaClosedChar(StateInPumpableRegexp prev, string char, StateInPumpableRegexp next) { + deltaClosed(prev, getAnInputSymbolMatchingRelevant(char), next) + } + + pragma[noinline] + InputSymbol getAnInputSymbolMatchingRelevant(string char) { + char = relevant(_) and + result = getAnInputSymbolMatching(char) + } + + /** + * Gets a char used for finding possible suffixes inside `root`. + */ + pragma[noinline] + private string relevant(RegExpRoot root) { + exists(ascii(result)) and exists(root) + or + exists(InputSymbol s | belongsTo(s, root) | result = intersect(s, _)) + or + // The characters from `hasSimpleRejectEdge`. Only `\n` is really needed (as `\n` is not in the `ascii` relation). + // The three chars must be kept in sync with `hasSimpleRejectEdge`. + result = ["|", "\n", "Z"] and exists(root) + } + + /** + * Holds if there exists a `char` such that there is no edge from `s` labeled `char` in our NFA. + * The NFA does not model reject states, so the above is the same as saying there is a reject edge. + */ + private predicate hasRejectEdge(State s) { + hasSimpleRejectEdge(s) + or + not hasSimpleRejectEdge(s) and + exists(string char | char = relevant(getRoot(s.getRepr())) | not deltaClosedChar(s, char, _)) + } + + /** + * Holds if there is no edge from `s` labeled with "|", "\n", or "Z" in our NFA. + * This predicate is used as a cheap pre-processing to speed up `hasRejectEdge`. + */ + private predicate hasSimpleRejectEdge(State s) { + // The three chars were chosen arbitrarily. The three chars must be kept in sync with `relevant`. + exists(string char | char = ["|", "\n", "Z"] | not deltaClosedChar(s, char, _)) + } + + /** + * Gets a state that can be reached from pumpable `fork` consuming all + * chars in `w` any number of times followed by the first `i+1` characters of `w`. + */ + pragma[noopt] + private State process(State fork, string w, int i) { + exists(State prev | prev = getProcessPrevious(fork, i, w) | + exists(string char, InputSymbol sym | + char = w.charAt(i) and + deltaClosed(prev, sym, result) and + // noopt to prevent joining `prev` with all possible `chars` that could transition away from `prev`. + // Instead only join with the set of `chars` where a relevant `InputSymbol` has already been found. + sym = getAProcessInputSymbol(char) + ) + ) + } + + /** + * Gets a state that can be reached from pumpable `fork` consuming all + * chars in `w` any number of times followed by the first `i` characters of `w`. + */ + private State getProcessPrevious(State fork, int i, string w) { + isReDoSCandidate(fork, w) and + ( + i = 0 and result = fork + or + result = process(fork, w, i - 1) + or + // repeat until fixpoint + i = 0 and + result = process(fork, w, w.length() - 1) + ) + } + + /** + * Gets an InputSymbol that matches `char`. + * The predicate is specialized to only have a result for the `char`s that are relevant for the `process` predicate. + */ + private InputSymbol getAProcessInputSymbol(string char) { + char = getAProcessChar() and + result = getAnInputSymbolMatching(char) + } + + /** + * Gets a `char` that occurs in a `pump` string. + */ + private string getAProcessChar() { result = any(string s | isReDoSCandidate(_, s)).charAt(_) } + } + + /** + * Holds if `term` may cause superlinear backtracking on strings containing many repetitions of `pump`. + * Gets the shortest string that causes superlinear backtracking. + */ + private predicate isReDoSAttackable(RegExpTerm term, string pump, State s) { + exists(int i, string c | s = Match(term, i) | + c = + min(string w | + any(ReDoSConfiguration conf).isReDoSCandidate(s, w) and + SuffixConstruction::reachesOnlyRejectableSuffixes(s, w) + | + w order by w.length(), w + ) and + pump = escape(rotate(c, i)) + ) + } + + /** + * Holds if the state `s` (represented by the term `t`) can have backtracking with repetitions of `pump`. + * + * `prefixMsg` contains a friendly message for a prefix that reaches `s` (or `prefixMsg` is the empty string if the prefix is empty or if no prefix could be found). + */ + predicate hasReDoSResult(RegExpTerm t, string pump, State s, string prefixMsg) { + isReDoSAttackable(t, pump, s) and + ( + prefixMsg = "starting with '" + escape(PrefixConstruction::prefix(s)) + "' and " and + not PrefixConstruction::prefix(s) = "" + or + PrefixConstruction::prefix(s) = "" and prefixMsg = "" + or + not exists(PrefixConstruction::prefix(s)) and prefixMsg = "" + ) + } } /** @@ -1162,37 +1198,3 @@ bindingset[str, i] private string rotate(string str, int i) { result = str.suffix(str.length() - i) + str.prefix(str.length() - i) } - -/** - * Holds if `term` may cause superlinear backtracking on strings containing many repetitions of `pump`. - * Gets the shortest string that causes superlinear backtracking. - */ -private predicate isReDoSAttackable(RegExpTerm term, string pump, State s) { - exists(int i, string c | s = Match(term, i) | - c = - min(string w | - any(ReDoSConfiguration conf).isReDoSCandidate(s, w) and - SuffixConstruction::reachesOnlyRejectableSuffixes(s, w) - | - w order by w.length(), w - ) and - pump = escape(rotate(c, i)) - ) -} - -/** - * Holds if the state `s` (represented by the term `t`) can have backtracking with repetitions of `pump`. - * - * `prefixMsg` contains a friendly message for a prefix that reaches `s` (or `prefixMsg` is the empty string if the prefix is empty or if no prefix could be found). - */ -predicate hasReDoSResult(RegExpTerm t, string pump, State s, string prefixMsg) { - isReDoSAttackable(t, pump, s) and - ( - prefixMsg = "starting with '" + escape(PrefixConstruction::prefix(s)) + "' and " and - not PrefixConstruction::prefix(s) = "" - or - PrefixConstruction::prefix(s) = "" and prefixMsg = "" - or - not exists(PrefixConstruction::prefix(s)) and prefixMsg = "" - ) -} diff --git a/python/ql/lib/semmle/python/security/performance/SuperlinearBackTracking.qll b/python/ql/lib/semmle/python/security/performance/SuperlinearBackTracking.qll index 4ba9520cdcc..05edbfd9777 100644 --- a/python/ql/lib/semmle/python/security/performance/SuperlinearBackTracking.qll +++ b/python/ql/lib/semmle/python/security/performance/SuperlinearBackTracking.qll @@ -391,7 +391,7 @@ predicate isPumpable(State pivot, State succ, string pump) { */ predicate polynimalReDoS(RegExpTerm t, string pump, string prefixMsg, RegExpTerm prev) { exists(State s, State pivot | - hasReDoSResult(t, pump, s, prefixMsg) and + ReDoSPruning::hasReDoSResult(t, pump, s, prefixMsg) and isPumpable(pivot, s, _) and prev = pivot.getRepr() ) diff --git a/python/ql/src/Security/CWE-730/ReDoS.ql b/python/ql/src/Security/CWE-730/ReDoS.ql index c6d5397e771..03b39686f1c 100644 --- a/python/ql/src/Security/CWE-730/ReDoS.ql +++ b/python/ql/src/Security/CWE-730/ReDoS.ql @@ -18,7 +18,7 @@ import semmle.python.security.performance.ExponentialBackTracking from RegExpTerm t, string pump, State s, string prefixMsg where - hasReDoSResult(t, pump, s, prefixMsg) and + ReDoSPruning::hasReDoSResult(t, pump, s, prefixMsg) and // exclude verbose mode regexes for now not t.getRegex().getAMode() = "VERBOSE" select t, diff --git a/ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtil.qll b/ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtil.qll index 8aa348bf62f..8315d2b8225 100644 --- a/ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtil.qll +++ b/ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtil.qll @@ -14,45 +14,6 @@ import ReDoSUtilSpecific -/** - * A configuration for which parts of a regular expression should be considered relevant for - * the different predicates in `ReDoS.qll`. - * Used to adjust the computations for either superlinear or exponential backtracking. - */ -abstract class ReDoSConfiguration extends string { - bindingset[this] - ReDoSConfiguration() { any() } - - /** - * Holds if `state` with the pump string `pump` is a candidate for a - * ReDoS vulnerable state. - * This is used to determine which states are considered for the prefix/suffix construction. - */ - abstract predicate isReDoSCandidate(State state, string pump); -} - -/** - * Holds if repeating `pump` starting at `state` is a candidate for causing backtracking. - * No check whether a rejected suffix exists has been made. - */ -private predicate isReDoSCandidate(State state, string pump) { - any(ReDoSConfiguration conf).isReDoSCandidate(state, pump) and - ( - not any(ReDoSConfiguration conf).isReDoSCandidate(epsilonSucc+(state), _) - or - epsilonSucc+(state) = state and - state = - max(State s, Location l | - s = epsilonSucc+(state) and - l = s.getRepr().getLocation() and - any(ReDoSConfiguration conf).isReDoSCandidate(s, _) and - s.getRepr() instanceof InfiniteRepetitionQuantifier - | - s order by l.getStartLine(), l.getStartColumn(), l.getEndColumn(), l.getEndLine() - ) - ) -} - /** * Gets the char after `c` (from a simplified ASCII table). */ @@ -877,266 +838,341 @@ predicate isStartState(State state) { } /** - * Predicates for constructing a prefix string that leads to a given state. + * A configuration for which parts of a regular expression should be considered relevant for + * the different predicates in `ReDoS.qll`. + * Used to adjust the computations for either superlinear or exponential backtracking. */ -private module PrefixConstruction { - /** - * Holds if `state` is the textually last start state for the regular expression. - */ - private predicate lastStartState(State state) { - exists(RegExpRoot root | - state = - max(StateInPumpableRegexp s, Location l | - isStartState(s) and getRoot(s.getRepr()) = root and l = s.getRepr().getLocation() - | - s - order by - l.getStartLine(), l.getStartColumn(), s.getRepr().toString(), l.getEndColumn(), - l.getEndLine() - ) - ) - } +abstract class ReDoSConfiguration extends string { + bindingset[this] + ReDoSConfiguration() { any() } /** - * Holds if there exists any transition (Epsilon() or other) from `a` to `b`. + * Holds if `state` with the pump string `pump` is a candidate for a + * ReDoS vulnerable state. + * This is used to determine which states are considered for the prefix/suffix construction. */ - private predicate existsTransition(State a, State b) { delta(a, _, b) } - - /** - * Gets the minimum number of transitions it takes to reach `state` from the `start` state. - */ - int prefixLength(State start, State state) = - shortestDistances(lastStartState/1, existsTransition/2)(start, state, result) - - /** - * Gets the minimum number of transitions it takes to reach `state` from the start state. - */ - private int lengthFromStart(State state) { result = prefixLength(_, state) } - - /** - * Gets a string for which the regular expression will reach `state`. - * - * Has at most one result for any given `state`. - * This predicate will not always have a result even if there is a ReDoS issue in - * the regular expression. - */ - string prefix(State state) { - lastStartState(state) and - result = "" - or - // the search stops past the last redos candidate state. - lengthFromStart(state) <= max(lengthFromStart(any(State s | isReDoSCandidate(s, _)))) and - exists(State prev | - // select a unique predecessor (by an arbitrary measure) - prev = - min(State s, Location loc | - lengthFromStart(s) = lengthFromStart(state) - 1 and - loc = s.getRepr().getLocation() and - delta(s, _, state) - | - s - order by - loc.getStartLine(), loc.getStartColumn(), loc.getEndLine(), loc.getEndColumn(), - s.getRepr().toString() - ) - | - // greedy search for the shortest prefix - result = prefix(prev) and delta(prev, Epsilon(), state) - or - not delta(prev, Epsilon(), state) and - result = prefix(prev) + getCanonicalEdgeChar(prev, state) - ) - } - - /** - * Gets a canonical char for which there exists a transition from `prev` to `next` in the NFA. - */ - private string getCanonicalEdgeChar(State prev, State next) { - result = - min(string c | delta(prev, any(InputSymbol symbol | c = intersect(Any(), symbol)), next)) - } - - /** - * A state within a regular expression that has a pumpable state. - */ - class StateInPumpableRegexp extends State { - pragma[noinline] - StateInPumpableRegexp() { - exists(State s | isReDoSCandidate(s, _) | getRoot(s.getRepr()) = getRoot(this.getRepr())) - } - } + abstract predicate isReDoSCandidate(State state, string pump); } -/** - * Predicates for testing the presence of a rejecting suffix. - * - * These predicates are used to ensure that the all states reached from the fork - * by repeating `w` have a rejecting suffix. - * - * For example, a regexp like `/^(a+)+/` will accept any string as long the prefix is - * some number of `"a"`s, and it is therefore not possible to construct a rejecting suffix. - * - * A regexp like `/(a+)+$/` or `/(a+)+b/` trivially has a rejecting suffix, - * as the suffix "X" will cause both the regular expressions to be rejected. - * - * The string `w` is repeated any number of times because it needs to be - * infinitely repeatedable for the attack to work. - * For the regular expression `/((ab)+)*abab/` the accepting state is not reachable from the fork - * using epsilon transitions. But any attempt at repeating `w` will end in a state that accepts all suffixes. - */ -private module SuffixConstruction { - import PrefixConstruction - +module ReDoSPruning { /** - * Holds if all states reachable from `fork` by repeating `w` - * are likely rejectable by appending some suffix. + * Holds if repeating `pump` starting at `state` is a candidate for causing backtracking. + * No check whether a rejected suffix exists has been made. */ - predicate reachesOnlyRejectableSuffixes(State fork, string w) { - isReDoSCandidate(fork, w) and - forex(State next | next = process(fork, w, w.length() - 1) | isLikelyRejectable(next)) - } - - /** - * Holds if there likely exists a suffix starting from `s` that leads to the regular expression being rejected. - * This predicate might find impossible suffixes when searching for suffixes of length > 1, which can cause FPs. - */ - pragma[noinline] - private predicate isLikelyRejectable(StateInPumpableRegexp s) { - // exists a reject edge with some char. - hasRejectEdge(s) - or - hasEdgeToLikelyRejectable(s) - or - // stopping here is rejection - isRejectState(s) - } - - /** - * Holds if `s` is not an accept state, and there is no epsilon transition to an accept state. - */ - predicate isRejectState(StateInPumpableRegexp s) { not epsilonSucc*(s) = Accept(_) } - - /** - * Holds if there is likely a non-empty suffix leading to rejection starting in `s`. - */ - pragma[noopt] - predicate hasEdgeToLikelyRejectable(StateInPumpableRegexp s) { - // all edges (at least one) with some char leads to another state that is rejectable. - // the `next` states might not share a common suffix, which can cause FPs. - exists(string char | char = hasEdgeToLikelyRejectableHelper(s) | - // noopt to force `hasEdgeToLikelyRejectableHelper` to be first in the join-order. - exists(State next | deltaClosedChar(s, char, next) | isLikelyRejectable(next)) and - forall(State next | deltaClosedChar(s, char, next) | isLikelyRejectable(next)) - ) - } - - /** - * Gets a char for there exists a transition away from `s`, - * and `s` has not been found to be rejectable by `hasRejectEdge` or `isRejectState`. - */ - pragma[noinline] - private string hasEdgeToLikelyRejectableHelper(StateInPumpableRegexp s) { - not hasRejectEdge(s) and - not isRejectState(s) and - deltaClosedChar(s, result, _) - } - - /** - * Holds if there is a state `next` that can be reached from `prev` - * along epsilon edges, such that there is a transition from - * `prev` to `next` that the character symbol `char`. - */ - predicate deltaClosedChar(StateInPumpableRegexp prev, string char, StateInPumpableRegexp next) { - deltaClosed(prev, getAnInputSymbolMatchingRelevant(char), next) - } - - pragma[noinline] - InputSymbol getAnInputSymbolMatchingRelevant(string char) { - char = relevant(_) and - result = getAnInputSymbolMatching(char) - } - - /** - * Gets a char used for finding possible suffixes inside `root`. - */ - pragma[noinline] - private string relevant(RegExpRoot root) { - exists(ascii(result)) and exists(root) - or - exists(InputSymbol s | belongsTo(s, root) | result = intersect(s, _)) - or - // The characters from `hasSimpleRejectEdge`. Only `\n` is really needed (as `\n` is not in the `ascii` relation). - // The three chars must be kept in sync with `hasSimpleRejectEdge`. - result = ["|", "\n", "Z"] and exists(root) - } - - /** - * Holds if there exists a `char` such that there is no edge from `s` labeled `char` in our NFA. - * The NFA does not model reject states, so the above is the same as saying there is a reject edge. - */ - private predicate hasRejectEdge(State s) { - hasSimpleRejectEdge(s) - or - not hasSimpleRejectEdge(s) and - exists(string char | char = relevant(getRoot(s.getRepr())) | not deltaClosedChar(s, char, _)) - } - - /** - * Holds if there is no edge from `s` labeled with "|", "\n", or "Z" in our NFA. - * This predicate is used as a cheap pre-processing to speed up `hasRejectEdge`. - */ - private predicate hasSimpleRejectEdge(State s) { - // The three chars were chosen arbitrarily. The three chars must be kept in sync with `relevant`. - exists(string char | char = ["|", "\n", "Z"] | not deltaClosedChar(s, char, _)) - } - - /** - * Gets a state that can be reached from pumpable `fork` consuming all - * chars in `w` any number of times followed by the first `i+1` characters of `w`. - */ - pragma[noopt] - private State process(State fork, string w, int i) { - exists(State prev | prev = getProcessPrevious(fork, i, w) | - exists(string char, InputSymbol sym | - char = w.charAt(i) and - deltaClosed(prev, sym, result) and - // noopt to prevent joining `prev` with all possible `chars` that could transition away from `prev`. - // Instead only join with the set of `chars` where a relevant `InputSymbol` has already been found. - sym = getAProcessInputSymbol(char) - ) - ) - } - - /** - * Gets a state that can be reached from pumpable `fork` consuming all - * chars in `w` any number of times followed by the first `i` characters of `w`. - */ - private State getProcessPrevious(State fork, int i, string w) { - isReDoSCandidate(fork, w) and + private predicate isReDoSCandidate(State state, string pump) { + any(ReDoSConfiguration conf).isReDoSCandidate(state, pump) and ( - i = 0 and result = fork + not any(ReDoSConfiguration conf).isReDoSCandidate(epsilonSucc+(state), _) or - result = process(fork, w, i - 1) - or - // repeat until fixpoint - i = 0 and - result = process(fork, w, w.length() - 1) + epsilonSucc+(state) = state and + state = + max(State s, Location l | + s = epsilonSucc+(state) and + l = s.getRepr().getLocation() and + any(ReDoSConfiguration conf).isReDoSCandidate(s, _) and + s.getRepr() instanceof InfiniteRepetitionQuantifier + | + s order by l.getStartLine(), l.getStartColumn(), l.getEndColumn(), l.getEndLine() + ) ) } /** - * Gets an InputSymbol that matches `char`. - * The predicate is specialized to only have a result for the `char`s that are relevant for the `process` predicate. + * Predicates for constructing a prefix string that leads to a given state. */ - private InputSymbol getAProcessInputSymbol(string char) { - char = getAProcessChar() and - result = getAnInputSymbolMatching(char) + private module PrefixConstruction { + /** + * Holds if `state` is the textually last start state for the regular expression. + */ + private predicate lastStartState(State state) { + exists(RegExpRoot root | + state = + max(StateInPumpableRegexp s, Location l | + isStartState(s) and getRoot(s.getRepr()) = root and l = s.getRepr().getLocation() + | + s + order by + l.getStartLine(), l.getStartColumn(), s.getRepr().toString(), l.getEndColumn(), + l.getEndLine() + ) + ) + } + + /** + * Holds if there exists any transition (Epsilon() or other) from `a` to `b`. + */ + private predicate existsTransition(State a, State b) { delta(a, _, b) } + + /** + * Gets the minimum number of transitions it takes to reach `state` from the `start` state. + */ + int prefixLength(State start, State state) = + shortestDistances(lastStartState/1, existsTransition/2)(start, state, result) + + /** + * Gets the minimum number of transitions it takes to reach `state` from the start state. + */ + private int lengthFromStart(State state) { result = prefixLength(_, state) } + + /** + * Gets a string for which the regular expression will reach `state`. + * + * Has at most one result for any given `state`. + * This predicate will not always have a result even if there is a ReDoS issue in + * the regular expression. + */ + string prefix(State state) { + lastStartState(state) and + result = "" + or + // the search stops past the last redos candidate state. + lengthFromStart(state) <= max(lengthFromStart(any(State s | isReDoSCandidate(s, _)))) and + exists(State prev | + // select a unique predecessor (by an arbitrary measure) + prev = + min(State s, Location loc | + lengthFromStart(s) = lengthFromStart(state) - 1 and + loc = s.getRepr().getLocation() and + delta(s, _, state) + | + s + order by + loc.getStartLine(), loc.getStartColumn(), loc.getEndLine(), loc.getEndColumn(), + s.getRepr().toString() + ) + | + // greedy search for the shortest prefix + result = prefix(prev) and delta(prev, Epsilon(), state) + or + not delta(prev, Epsilon(), state) and + result = prefix(prev) + getCanonicalEdgeChar(prev, state) + ) + } + + /** + * Gets a canonical char for which there exists a transition from `prev` to `next` in the NFA. + */ + private string getCanonicalEdgeChar(State prev, State next) { + result = + min(string c | delta(prev, any(InputSymbol symbol | c = intersect(Any(), symbol)), next)) + } + + /** + * A state within a regular expression that has a pumpable state. + */ + class StateInPumpableRegexp extends State { + pragma[noinline] + StateInPumpableRegexp() { + exists(State s | isReDoSCandidate(s, _) | getRoot(s.getRepr()) = getRoot(this.getRepr())) + } + } } /** - * Gets a `char` that occurs in a `pump` string. + * Predicates for testing the presence of a rejecting suffix. + * + * These predicates are used to ensure that the all states reached from the fork + * by repeating `w` have a rejecting suffix. + * + * For example, a regexp like `/^(a+)+/` will accept any string as long the prefix is + * some number of `"a"`s, and it is therefore not possible to construct a rejecting suffix. + * + * A regexp like `/(a+)+$/` or `/(a+)+b/` trivially has a rejecting suffix, + * as the suffix "X" will cause both the regular expressions to be rejected. + * + * The string `w` is repeated any number of times because it needs to be + * infinitely repeatedable for the attack to work. + * For the regular expression `/((ab)+)*abab/` the accepting state is not reachable from the fork + * using epsilon transitions. But any attempt at repeating `w` will end in a state that accepts all suffixes. */ - private string getAProcessChar() { result = any(string s | isReDoSCandidate(_, s)).charAt(_) } + private module SuffixConstruction { + import PrefixConstruction + + /** + * Holds if all states reachable from `fork` by repeating `w` + * are likely rejectable by appending some suffix. + */ + predicate reachesOnlyRejectableSuffixes(State fork, string w) { + isReDoSCandidate(fork, w) and + forex(State next | next = process(fork, w, w.length() - 1) | isLikelyRejectable(next)) + } + + /** + * Holds if there likely exists a suffix starting from `s` that leads to the regular expression being rejected. + * This predicate might find impossible suffixes when searching for suffixes of length > 1, which can cause FPs. + */ + pragma[noinline] + private predicate isLikelyRejectable(StateInPumpableRegexp s) { + // exists a reject edge with some char. + hasRejectEdge(s) + or + hasEdgeToLikelyRejectable(s) + or + // stopping here is rejection + isRejectState(s) + } + + /** + * Holds if `s` is not an accept state, and there is no epsilon transition to an accept state. + */ + predicate isRejectState(StateInPumpableRegexp s) { not epsilonSucc*(s) = Accept(_) } + + /** + * Holds if there is likely a non-empty suffix leading to rejection starting in `s`. + */ + pragma[noopt] + predicate hasEdgeToLikelyRejectable(StateInPumpableRegexp s) { + // all edges (at least one) with some char leads to another state that is rejectable. + // the `next` states might not share a common suffix, which can cause FPs. + exists(string char | char = hasEdgeToLikelyRejectableHelper(s) | + // noopt to force `hasEdgeToLikelyRejectableHelper` to be first in the join-order. + exists(State next | deltaClosedChar(s, char, next) | isLikelyRejectable(next)) and + forall(State next | deltaClosedChar(s, char, next) | isLikelyRejectable(next)) + ) + } + + /** + * Gets a char for there exists a transition away from `s`, + * and `s` has not been found to be rejectable by `hasRejectEdge` or `isRejectState`. + */ + pragma[noinline] + private string hasEdgeToLikelyRejectableHelper(StateInPumpableRegexp s) { + not hasRejectEdge(s) and + not isRejectState(s) and + deltaClosedChar(s, result, _) + } + + /** + * Holds if there is a state `next` that can be reached from `prev` + * along epsilon edges, such that there is a transition from + * `prev` to `next` that the character symbol `char`. + */ + predicate deltaClosedChar(StateInPumpableRegexp prev, string char, StateInPumpableRegexp next) { + deltaClosed(prev, getAnInputSymbolMatchingRelevant(char), next) + } + + pragma[noinline] + InputSymbol getAnInputSymbolMatchingRelevant(string char) { + char = relevant(_) and + result = getAnInputSymbolMatching(char) + } + + /** + * Gets a char used for finding possible suffixes inside `root`. + */ + pragma[noinline] + private string relevant(RegExpRoot root) { + exists(ascii(result)) and exists(root) + or + exists(InputSymbol s | belongsTo(s, root) | result = intersect(s, _)) + or + // The characters from `hasSimpleRejectEdge`. Only `\n` is really needed (as `\n` is not in the `ascii` relation). + // The three chars must be kept in sync with `hasSimpleRejectEdge`. + result = ["|", "\n", "Z"] and exists(root) + } + + /** + * Holds if there exists a `char` such that there is no edge from `s` labeled `char` in our NFA. + * The NFA does not model reject states, so the above is the same as saying there is a reject edge. + */ + private predicate hasRejectEdge(State s) { + hasSimpleRejectEdge(s) + or + not hasSimpleRejectEdge(s) and + exists(string char | char = relevant(getRoot(s.getRepr())) | not deltaClosedChar(s, char, _)) + } + + /** + * Holds if there is no edge from `s` labeled with "|", "\n", or "Z" in our NFA. + * This predicate is used as a cheap pre-processing to speed up `hasRejectEdge`. + */ + private predicate hasSimpleRejectEdge(State s) { + // The three chars were chosen arbitrarily. The three chars must be kept in sync with `relevant`. + exists(string char | char = ["|", "\n", "Z"] | not deltaClosedChar(s, char, _)) + } + + /** + * Gets a state that can be reached from pumpable `fork` consuming all + * chars in `w` any number of times followed by the first `i+1` characters of `w`. + */ + pragma[noopt] + private State process(State fork, string w, int i) { + exists(State prev | prev = getProcessPrevious(fork, i, w) | + exists(string char, InputSymbol sym | + char = w.charAt(i) and + deltaClosed(prev, sym, result) and + // noopt to prevent joining `prev` with all possible `chars` that could transition away from `prev`. + // Instead only join with the set of `chars` where a relevant `InputSymbol` has already been found. + sym = getAProcessInputSymbol(char) + ) + ) + } + + /** + * Gets a state that can be reached from pumpable `fork` consuming all + * chars in `w` any number of times followed by the first `i` characters of `w`. + */ + private State getProcessPrevious(State fork, int i, string w) { + isReDoSCandidate(fork, w) and + ( + i = 0 and result = fork + or + result = process(fork, w, i - 1) + or + // repeat until fixpoint + i = 0 and + result = process(fork, w, w.length() - 1) + ) + } + + /** + * Gets an InputSymbol that matches `char`. + * The predicate is specialized to only have a result for the `char`s that are relevant for the `process` predicate. + */ + private InputSymbol getAProcessInputSymbol(string char) { + char = getAProcessChar() and + result = getAnInputSymbolMatching(char) + } + + /** + * Gets a `char` that occurs in a `pump` string. + */ + private string getAProcessChar() { result = any(string s | isReDoSCandidate(_, s)).charAt(_) } + } + + /** + * Holds if `term` may cause superlinear backtracking on strings containing many repetitions of `pump`. + * Gets the shortest string that causes superlinear backtracking. + */ + private predicate isReDoSAttackable(RegExpTerm term, string pump, State s) { + exists(int i, string c | s = Match(term, i) | + c = + min(string w | + any(ReDoSConfiguration conf).isReDoSCandidate(s, w) and + SuffixConstruction::reachesOnlyRejectableSuffixes(s, w) + | + w order by w.length(), w + ) and + pump = escape(rotate(c, i)) + ) + } + + /** + * Holds if the state `s` (represented by the term `t`) can have backtracking with repetitions of `pump`. + * + * `prefixMsg` contains a friendly message for a prefix that reaches `s` (or `prefixMsg` is the empty string if the prefix is empty or if no prefix could be found). + */ + predicate hasReDoSResult(RegExpTerm t, string pump, State s, string prefixMsg) { + isReDoSAttackable(t, pump, s) and + ( + prefixMsg = "starting with '" + escape(PrefixConstruction::prefix(s)) + "' and " and + not PrefixConstruction::prefix(s) = "" + or + PrefixConstruction::prefix(s) = "" and prefixMsg = "" + or + not exists(PrefixConstruction::prefix(s)) and prefixMsg = "" + ) + } } /** @@ -1162,37 +1198,3 @@ bindingset[str, i] private string rotate(string str, int i) { result = str.suffix(str.length() - i) + str.prefix(str.length() - i) } - -/** - * Holds if `term` may cause superlinear backtracking on strings containing many repetitions of `pump`. - * Gets the shortest string that causes superlinear backtracking. - */ -private predicate isReDoSAttackable(RegExpTerm term, string pump, State s) { - exists(int i, string c | s = Match(term, i) | - c = - min(string w | - any(ReDoSConfiguration conf).isReDoSCandidate(s, w) and - SuffixConstruction::reachesOnlyRejectableSuffixes(s, w) - | - w order by w.length(), w - ) and - pump = escape(rotate(c, i)) - ) -} - -/** - * Holds if the state `s` (represented by the term `t`) can have backtracking with repetitions of `pump`. - * - * `prefixMsg` contains a friendly message for a prefix that reaches `s` (or `prefixMsg` is the empty string if the prefix is empty or if no prefix could be found). - */ -predicate hasReDoSResult(RegExpTerm t, string pump, State s, string prefixMsg) { - isReDoSAttackable(t, pump, s) and - ( - prefixMsg = "starting with '" + escape(PrefixConstruction::prefix(s)) + "' and " and - not PrefixConstruction::prefix(s) = "" - or - PrefixConstruction::prefix(s) = "" and prefixMsg = "" - or - not exists(PrefixConstruction::prefix(s)) and prefixMsg = "" - ) -} diff --git a/ruby/ql/lib/codeql/ruby/security/performance/SuperlinearBackTracking.qll b/ruby/ql/lib/codeql/ruby/security/performance/SuperlinearBackTracking.qll index 4ba9520cdcc..05edbfd9777 100644 --- a/ruby/ql/lib/codeql/ruby/security/performance/SuperlinearBackTracking.qll +++ b/ruby/ql/lib/codeql/ruby/security/performance/SuperlinearBackTracking.qll @@ -391,7 +391,7 @@ predicate isPumpable(State pivot, State succ, string pump) { */ predicate polynimalReDoS(RegExpTerm t, string pump, string prefixMsg, RegExpTerm prev) { exists(State s, State pivot | - hasReDoSResult(t, pump, s, prefixMsg) and + ReDoSPruning::hasReDoSResult(t, pump, s, prefixMsg) and isPumpable(pivot, s, _) and prev = pivot.getRepr() ) diff --git a/ruby/ql/src/queries/security/cwe-1333/ReDoS.ql b/ruby/ql/src/queries/security/cwe-1333/ReDoS.ql index e9ccce7fd04..4491b79da2a 100644 --- a/ruby/ql/src/queries/security/cwe-1333/ReDoS.ql +++ b/ruby/ql/src/queries/security/cwe-1333/ReDoS.ql @@ -19,7 +19,7 @@ import codeql.ruby.security.performance.ReDoSUtil import codeql.ruby.Regexp from RegExpTerm t, string pump, State s, string prefixMsg -where hasReDoSResult(t, pump, s, prefixMsg) +where ReDoSPruning::hasReDoSResult(t, pump, s, prefixMsg) select t, "This part of the regular expression may cause exponential backtracking on strings " + prefixMsg + "containing many repetitions of '" + pump + "'." From 3be4a86acd4cad535816db8385925a081a2312af Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Mon, 14 Feb 2022 13:26:08 +0100 Subject: [PATCH 002/600] make ReDoSPruning into a parameterized module --- .../performance/ExponentialBackTracking.qll | 10 +- .../java/security/performance/ReDoSUtil.qll | 128 +++++++++--------- .../performance/SuperlinearBackTracking.qll | 16 +-- .../query-tests/security/CWE-730/ReDoS.ql | 2 +- .../performance/ExponentialBackTracking.qll | 10 +- .../security/performance/ReDoSUtil.qll | 128 +++++++++--------- .../performance/SuperlinearBackTracking.qll | 16 +-- javascript/ql/src/Performance/ReDoS.ql | 2 +- .../performance/ExponentialBackTracking.qll | 10 +- .../python/security/performance/ReDoSUtil.qll | 128 +++++++++--------- .../performance/SuperlinearBackTracking.qll | 16 +-- python/ql/src/Security/CWE-730/ReDoS.ql | 2 +- .../performance/ExponentialBackTracking.qll | 10 +- .../ruby/security/performance/ReDoSUtil.qll | 128 +++++++++--------- .../performance/SuperlinearBackTracking.qll | 16 +-- .../ql/src/queries/security/cwe-1333/ReDoS.ql | 2 +- 16 files changed, 296 insertions(+), 328 deletions(-) diff --git a/java/ql/lib/semmle/code/java/security/performance/ExponentialBackTracking.qll b/java/ql/lib/semmle/code/java/security/performance/ExponentialBackTracking.qll index 8ba5a75d431..71dff93acb0 100644 --- a/java/ql/lib/semmle/code/java/security/performance/ExponentialBackTracking.qll +++ b/java/ql/lib/semmle/code/java/security/performance/ExponentialBackTracking.qll @@ -364,11 +364,5 @@ private predicate isPumpable(State fork, string w) { ) } -/** - * An instantiation of `ReDoSConfiguration` for exponential backtracking. - */ -class ExponentialReDoSConfiguration extends ReDoSConfiguration { - ExponentialReDoSConfiguration() { this = "ExponentialReDoSConfiguration" } - - override predicate isReDoSCandidate(State state, string pump) { isPumpable(state, pump) } -} +/** Holds if `state` has exponential ReDoS */ +predicate hasReDoSResult = ReDoSPruning::hasReDoSResult/4; diff --git a/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll b/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll index 8315d2b8225..35f8ffc0e65 100644 --- a/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll +++ b/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll @@ -838,38 +838,32 @@ predicate isStartState(State state) { } /** - * A configuration for which parts of a regular expression should be considered relevant for - * the different predicates in `ReDoS.qll`. - * Used to adjust the computations for either superlinear or exponential backtracking. + * Holds if `state` is a candidate for ReDoS with string `pump`. */ -abstract class ReDoSConfiguration extends string { - bindingset[this] - ReDoSConfiguration() { any() } +signature predicate isCandidateSig(State state, string pump); - /** - * Holds if `state` with the pump string `pump` is a candidate for a - * ReDoS vulnerable state. - * This is used to determine which states are considered for the prefix/suffix construction. - */ - abstract predicate isReDoSCandidate(State state, string pump); -} - -module ReDoSPruning { +/** + * A module for pruning candidate ReDoS states. + * The candidates are specified by the `isCandidate` signature predicate. + * The candidates are checked for rejecting suffixes and deduplicated, + * and the resulting ReDoS states are read by the `hasReDoSResult` predicate. + */ +module ReDoSPruning { /** * Holds if repeating `pump` starting at `state` is a candidate for causing backtracking. * No check whether a rejected suffix exists has been made. */ private predicate isReDoSCandidate(State state, string pump) { - any(ReDoSConfiguration conf).isReDoSCandidate(state, pump) and + isCandidate(state, pump) and ( - not any(ReDoSConfiguration conf).isReDoSCandidate(epsilonSucc+(state), _) + not isCandidate(epsilonSucc+(state), _) or epsilonSucc+(state) = state and state = max(State s, Location l | s = epsilonSucc+(state) and l = s.getRepr().getLocation() and - any(ReDoSConfiguration conf).isReDoSCandidate(s, _) and + isCandidate(s, _) and s.getRepr() instanceof InfiniteRepetitionQuantifier | s order by l.getStartLine(), l.getStartColumn(), l.getEndColumn(), l.getEndLine() @@ -887,8 +881,11 @@ module ReDoSPruning { private predicate lastStartState(State state) { exists(RegExpRoot root | state = - max(StateInPumpableRegexp s, Location l | - isStartState(s) and getRoot(s.getRepr()) = root and l = s.getRepr().getLocation() + max(State s, Location l | + s = stateInPumpableRegexp() and + isStartState(s) and + getRoot(s.getRepr()) = root and + l = s.getRepr().getLocation() | s order by @@ -957,14 +954,10 @@ module ReDoSPruning { min(string c | delta(prev, any(InputSymbol symbol | c = intersect(Any(), symbol)), next)) } - /** - * A state within a regular expression that has a pumpable state. - */ - class StateInPumpableRegexp extends State { - pragma[noinline] - StateInPumpableRegexp() { - exists(State s | isReDoSCandidate(s, _) | getRoot(s.getRepr()) = getRoot(this.getRepr())) - } + /** Gets a state within a regular expression that has a pumpable state. */ + pragma[noinline] + State stateInPumpableRegexp() { + exists(State s | isReDoSCandidate(s, _) | getRoot(s.getRepr()) = getRoot(result.getRepr())) } } @@ -1002,26 +995,32 @@ module ReDoSPruning { * This predicate might find impossible suffixes when searching for suffixes of length > 1, which can cause FPs. */ pragma[noinline] - private predicate isLikelyRejectable(StateInPumpableRegexp s) { - // exists a reject edge with some char. - hasRejectEdge(s) - or - hasEdgeToLikelyRejectable(s) - or - // stopping here is rejection - isRejectState(s) + private predicate isLikelyRejectable(State s) { + s = stateInPumpableRegexp() and + ( + // exists a reject edge with some char. + hasRejectEdge(s) + or + hasEdgeToLikelyRejectable(s) + or + // stopping here is rejection + isRejectState(s) + ) } /** * Holds if `s` is not an accept state, and there is no epsilon transition to an accept state. */ - predicate isRejectState(StateInPumpableRegexp s) { not epsilonSucc*(s) = Accept(_) } + predicate isRejectState(State s) { + s = stateInPumpableRegexp() and not epsilonSucc*(s) = Accept(_) + } /** * Holds if there is likely a non-empty suffix leading to rejection starting in `s`. */ pragma[noopt] - predicate hasEdgeToLikelyRejectable(StateInPumpableRegexp s) { + predicate hasEdgeToLikelyRejectable(State s) { + s = stateInPumpableRegexp() and // all edges (at least one) with some char leads to another state that is rejectable. // the `next` states might not share a common suffix, which can cause FPs. exists(string char | char = hasEdgeToLikelyRejectableHelper(s) | @@ -1036,7 +1035,8 @@ module ReDoSPruning { * and `s` has not been found to be rejectable by `hasRejectEdge` or `isRejectState`. */ pragma[noinline] - private string hasEdgeToLikelyRejectableHelper(StateInPumpableRegexp s) { + private string hasEdgeToLikelyRejectableHelper(State s) { + s = stateInPumpableRegexp() and not hasRejectEdge(s) and not isRejectState(s) and deltaClosedChar(s, result, _) @@ -1047,7 +1047,9 @@ module ReDoSPruning { * along epsilon edges, such that there is a transition from * `prev` to `next` that the character symbol `char`. */ - predicate deltaClosedChar(StateInPumpableRegexp prev, string char, StateInPumpableRegexp next) { + predicate deltaClosedChar(State prev, string char, State next) { + prev = stateInPumpableRegexp() and + next = stateInPumpableRegexp() and deltaClosed(prev, getAnInputSymbolMatchingRelevant(char), next) } @@ -1148,7 +1150,7 @@ module ReDoSPruning { exists(int i, string c | s = Match(term, i) | c = min(string w | - any(ReDoSConfiguration conf).isReDoSCandidate(s, w) and + isCandidate(s, w) and SuffixConstruction::reachesOnlyRejectableSuffixes(s, w) | w order by w.length(), w @@ -1173,28 +1175,28 @@ module ReDoSPruning { not exists(PrefixConstruction::prefix(s)) and prefixMsg = "" ) } -} -/** - * Gets the result of backslash-escaping newlines, carriage-returns and - * backslashes in `s`. - */ -bindingset[s] -private string escape(string s) { - result = - s.replaceAll("\\", "\\\\") - .replaceAll("\n", "\\n") - .replaceAll("\r", "\\r") - .replaceAll("\t", "\\t") -} + /** + * Gets the result of backslash-escaping newlines, carriage-returns and + * backslashes in `s`. + */ + bindingset[s] + private string escape(string s) { + result = + s.replaceAll("\\", "\\\\") + .replaceAll("\n", "\\n") + .replaceAll("\r", "\\r") + .replaceAll("\t", "\\t") + } -/** - * Gets `str` with the last `i` characters moved to the front. - * - * We use this to adjust the pump string to match with the beginning of - * a RegExpTerm, so it doesn't start in the middle of a constant. - */ -bindingset[str, i] -private string rotate(string str, int i) { - result = str.suffix(str.length() - i) + str.prefix(str.length() - i) + /** + * Gets `str` with the last `i` characters moved to the front. + * + * We use this to adjust the pump string to match with the beginning of + * a RegExpTerm, so it doesn't start in the middle of a constant. + */ + bindingset[str, i] + private string rotate(string str, int i) { + result = str.suffix(str.length() - i) + str.prefix(str.length() - i) + } } diff --git a/java/ql/lib/semmle/code/java/security/performance/SuperlinearBackTracking.qll b/java/ql/lib/semmle/code/java/security/performance/SuperlinearBackTracking.qll index 05edbfd9777..b1916cfd76b 100644 --- a/java/ql/lib/semmle/code/java/security/performance/SuperlinearBackTracking.qll +++ b/java/ql/lib/semmle/code/java/security/performance/SuperlinearBackTracking.qll @@ -42,15 +42,6 @@ import ReDoSUtil * It also doesn't find all transitions in the product automaton, which can cause false negatives. */ -/** - * An instantiaion of `ReDoSConfiguration` for superlinear ReDoS. - */ -class SuperLinearReDoSConfiguration extends ReDoSConfiguration { - SuperLinearReDoSConfiguration() { this = "SuperLinearReDoSConfiguration" } - - override predicate isReDoSCandidate(State state, string pump) { isPumpable(_, state, pump) } -} - /** * Gets any root (start) state of a regular expression. */ @@ -386,12 +377,17 @@ predicate isPumpable(State pivot, State succ, string pump) { ) } +/** + * Holds if states starting in `state` can have polynomial backtracking with the string `pump`. + */ +predicate isReDoSCandidate(State state, string pump) { isPumpable(_, state, pump) } + /** * Holds if repetitions of `pump` at `t` will cause polynomial backtracking. */ predicate polynimalReDoS(RegExpTerm t, string pump, string prefixMsg, RegExpTerm prev) { exists(State s, State pivot | - ReDoSPruning::hasReDoSResult(t, pump, s, prefixMsg) and + ReDoSPruning::hasReDoSResult(t, pump, s, prefixMsg) and isPumpable(pivot, s, _) and prev = pivot.getRepr() ) diff --git a/java/ql/test/query-tests/security/CWE-730/ReDoS.ql b/java/ql/test/query-tests/security/CWE-730/ReDoS.ql index ae85707fc18..79cb8243cd7 100644 --- a/java/ql/test/query-tests/security/CWE-730/ReDoS.ql +++ b/java/ql/test/query-tests/security/CWE-730/ReDoS.ql @@ -11,7 +11,7 @@ class HasExpRedos extends InlineExpectationsTest { override predicate hasActualResult(Location location, string element, string tag, string value) { tag = "hasExpRedos" and exists(RegExpTerm t, string pump, State s, string prefixMsg | - ReDoSPruning::hasReDoSResult(t, pump, s, prefixMsg) and + hasReDoSResult(t, pump, s, prefixMsg) and not t.getRegex().getAMode() = "VERBOSE" and value = "" and location = t.getLocation() and diff --git a/javascript/ql/lib/semmle/javascript/security/performance/ExponentialBackTracking.qll b/javascript/ql/lib/semmle/javascript/security/performance/ExponentialBackTracking.qll index 8ba5a75d431..71dff93acb0 100644 --- a/javascript/ql/lib/semmle/javascript/security/performance/ExponentialBackTracking.qll +++ b/javascript/ql/lib/semmle/javascript/security/performance/ExponentialBackTracking.qll @@ -364,11 +364,5 @@ private predicate isPumpable(State fork, string w) { ) } -/** - * An instantiation of `ReDoSConfiguration` for exponential backtracking. - */ -class ExponentialReDoSConfiguration extends ReDoSConfiguration { - ExponentialReDoSConfiguration() { this = "ExponentialReDoSConfiguration" } - - override predicate isReDoSCandidate(State state, string pump) { isPumpable(state, pump) } -} +/** Holds if `state` has exponential ReDoS */ +predicate hasReDoSResult = ReDoSPruning::hasReDoSResult/4; diff --git a/javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtil.qll b/javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtil.qll index 8315d2b8225..35f8ffc0e65 100644 --- a/javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtil.qll +++ b/javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtil.qll @@ -838,38 +838,32 @@ predicate isStartState(State state) { } /** - * A configuration for which parts of a regular expression should be considered relevant for - * the different predicates in `ReDoS.qll`. - * Used to adjust the computations for either superlinear or exponential backtracking. + * Holds if `state` is a candidate for ReDoS with string `pump`. */ -abstract class ReDoSConfiguration extends string { - bindingset[this] - ReDoSConfiguration() { any() } +signature predicate isCandidateSig(State state, string pump); - /** - * Holds if `state` with the pump string `pump` is a candidate for a - * ReDoS vulnerable state. - * This is used to determine which states are considered for the prefix/suffix construction. - */ - abstract predicate isReDoSCandidate(State state, string pump); -} - -module ReDoSPruning { +/** + * A module for pruning candidate ReDoS states. + * The candidates are specified by the `isCandidate` signature predicate. + * The candidates are checked for rejecting suffixes and deduplicated, + * and the resulting ReDoS states are read by the `hasReDoSResult` predicate. + */ +module ReDoSPruning { /** * Holds if repeating `pump` starting at `state` is a candidate for causing backtracking. * No check whether a rejected suffix exists has been made. */ private predicate isReDoSCandidate(State state, string pump) { - any(ReDoSConfiguration conf).isReDoSCandidate(state, pump) and + isCandidate(state, pump) and ( - not any(ReDoSConfiguration conf).isReDoSCandidate(epsilonSucc+(state), _) + not isCandidate(epsilonSucc+(state), _) or epsilonSucc+(state) = state and state = max(State s, Location l | s = epsilonSucc+(state) and l = s.getRepr().getLocation() and - any(ReDoSConfiguration conf).isReDoSCandidate(s, _) and + isCandidate(s, _) and s.getRepr() instanceof InfiniteRepetitionQuantifier | s order by l.getStartLine(), l.getStartColumn(), l.getEndColumn(), l.getEndLine() @@ -887,8 +881,11 @@ module ReDoSPruning { private predicate lastStartState(State state) { exists(RegExpRoot root | state = - max(StateInPumpableRegexp s, Location l | - isStartState(s) and getRoot(s.getRepr()) = root and l = s.getRepr().getLocation() + max(State s, Location l | + s = stateInPumpableRegexp() and + isStartState(s) and + getRoot(s.getRepr()) = root and + l = s.getRepr().getLocation() | s order by @@ -957,14 +954,10 @@ module ReDoSPruning { min(string c | delta(prev, any(InputSymbol symbol | c = intersect(Any(), symbol)), next)) } - /** - * A state within a regular expression that has a pumpable state. - */ - class StateInPumpableRegexp extends State { - pragma[noinline] - StateInPumpableRegexp() { - exists(State s | isReDoSCandidate(s, _) | getRoot(s.getRepr()) = getRoot(this.getRepr())) - } + /** Gets a state within a regular expression that has a pumpable state. */ + pragma[noinline] + State stateInPumpableRegexp() { + exists(State s | isReDoSCandidate(s, _) | getRoot(s.getRepr()) = getRoot(result.getRepr())) } } @@ -1002,26 +995,32 @@ module ReDoSPruning { * This predicate might find impossible suffixes when searching for suffixes of length > 1, which can cause FPs. */ pragma[noinline] - private predicate isLikelyRejectable(StateInPumpableRegexp s) { - // exists a reject edge with some char. - hasRejectEdge(s) - or - hasEdgeToLikelyRejectable(s) - or - // stopping here is rejection - isRejectState(s) + private predicate isLikelyRejectable(State s) { + s = stateInPumpableRegexp() and + ( + // exists a reject edge with some char. + hasRejectEdge(s) + or + hasEdgeToLikelyRejectable(s) + or + // stopping here is rejection + isRejectState(s) + ) } /** * Holds if `s` is not an accept state, and there is no epsilon transition to an accept state. */ - predicate isRejectState(StateInPumpableRegexp s) { not epsilonSucc*(s) = Accept(_) } + predicate isRejectState(State s) { + s = stateInPumpableRegexp() and not epsilonSucc*(s) = Accept(_) + } /** * Holds if there is likely a non-empty suffix leading to rejection starting in `s`. */ pragma[noopt] - predicate hasEdgeToLikelyRejectable(StateInPumpableRegexp s) { + predicate hasEdgeToLikelyRejectable(State s) { + s = stateInPumpableRegexp() and // all edges (at least one) with some char leads to another state that is rejectable. // the `next` states might not share a common suffix, which can cause FPs. exists(string char | char = hasEdgeToLikelyRejectableHelper(s) | @@ -1036,7 +1035,8 @@ module ReDoSPruning { * and `s` has not been found to be rejectable by `hasRejectEdge` or `isRejectState`. */ pragma[noinline] - private string hasEdgeToLikelyRejectableHelper(StateInPumpableRegexp s) { + private string hasEdgeToLikelyRejectableHelper(State s) { + s = stateInPumpableRegexp() and not hasRejectEdge(s) and not isRejectState(s) and deltaClosedChar(s, result, _) @@ -1047,7 +1047,9 @@ module ReDoSPruning { * along epsilon edges, such that there is a transition from * `prev` to `next` that the character symbol `char`. */ - predicate deltaClosedChar(StateInPumpableRegexp prev, string char, StateInPumpableRegexp next) { + predicate deltaClosedChar(State prev, string char, State next) { + prev = stateInPumpableRegexp() and + next = stateInPumpableRegexp() and deltaClosed(prev, getAnInputSymbolMatchingRelevant(char), next) } @@ -1148,7 +1150,7 @@ module ReDoSPruning { exists(int i, string c | s = Match(term, i) | c = min(string w | - any(ReDoSConfiguration conf).isReDoSCandidate(s, w) and + isCandidate(s, w) and SuffixConstruction::reachesOnlyRejectableSuffixes(s, w) | w order by w.length(), w @@ -1173,28 +1175,28 @@ module ReDoSPruning { not exists(PrefixConstruction::prefix(s)) and prefixMsg = "" ) } -} -/** - * Gets the result of backslash-escaping newlines, carriage-returns and - * backslashes in `s`. - */ -bindingset[s] -private string escape(string s) { - result = - s.replaceAll("\\", "\\\\") - .replaceAll("\n", "\\n") - .replaceAll("\r", "\\r") - .replaceAll("\t", "\\t") -} + /** + * Gets the result of backslash-escaping newlines, carriage-returns and + * backslashes in `s`. + */ + bindingset[s] + private string escape(string s) { + result = + s.replaceAll("\\", "\\\\") + .replaceAll("\n", "\\n") + .replaceAll("\r", "\\r") + .replaceAll("\t", "\\t") + } -/** - * Gets `str` with the last `i` characters moved to the front. - * - * We use this to adjust the pump string to match with the beginning of - * a RegExpTerm, so it doesn't start in the middle of a constant. - */ -bindingset[str, i] -private string rotate(string str, int i) { - result = str.suffix(str.length() - i) + str.prefix(str.length() - i) + /** + * Gets `str` with the last `i` characters moved to the front. + * + * We use this to adjust the pump string to match with the beginning of + * a RegExpTerm, so it doesn't start in the middle of a constant. + */ + bindingset[str, i] + private string rotate(string str, int i) { + result = str.suffix(str.length() - i) + str.prefix(str.length() - i) + } } diff --git a/javascript/ql/lib/semmle/javascript/security/performance/SuperlinearBackTracking.qll b/javascript/ql/lib/semmle/javascript/security/performance/SuperlinearBackTracking.qll index 05edbfd9777..b1916cfd76b 100644 --- a/javascript/ql/lib/semmle/javascript/security/performance/SuperlinearBackTracking.qll +++ b/javascript/ql/lib/semmle/javascript/security/performance/SuperlinearBackTracking.qll @@ -42,15 +42,6 @@ import ReDoSUtil * It also doesn't find all transitions in the product automaton, which can cause false negatives. */ -/** - * An instantiaion of `ReDoSConfiguration` for superlinear ReDoS. - */ -class SuperLinearReDoSConfiguration extends ReDoSConfiguration { - SuperLinearReDoSConfiguration() { this = "SuperLinearReDoSConfiguration" } - - override predicate isReDoSCandidate(State state, string pump) { isPumpable(_, state, pump) } -} - /** * Gets any root (start) state of a regular expression. */ @@ -386,12 +377,17 @@ predicate isPumpable(State pivot, State succ, string pump) { ) } +/** + * Holds if states starting in `state` can have polynomial backtracking with the string `pump`. + */ +predicate isReDoSCandidate(State state, string pump) { isPumpable(_, state, pump) } + /** * Holds if repetitions of `pump` at `t` will cause polynomial backtracking. */ predicate polynimalReDoS(RegExpTerm t, string pump, string prefixMsg, RegExpTerm prev) { exists(State s, State pivot | - ReDoSPruning::hasReDoSResult(t, pump, s, prefixMsg) and + ReDoSPruning::hasReDoSResult(t, pump, s, prefixMsg) and isPumpable(pivot, s, _) and prev = pivot.getRepr() ) diff --git a/javascript/ql/src/Performance/ReDoS.ql b/javascript/ql/src/Performance/ReDoS.ql index 854f3653e8d..bda773bf0e7 100644 --- a/javascript/ql/src/Performance/ReDoS.ql +++ b/javascript/ql/src/Performance/ReDoS.ql @@ -19,7 +19,7 @@ import semmle.javascript.security.performance.ReDoSUtil import semmle.javascript.security.performance.ExponentialBackTracking from RegExpTerm t, string pump, State s, string prefixMsg -where ReDoSPruning::hasReDoSResult(t, pump, s, prefixMsg) +where hasReDoSResult(t, pump, s, prefixMsg) select t, "This part of the regular expression may cause exponential backtracking on strings " + prefixMsg + "containing many repetitions of '" + pump + "'." diff --git a/python/ql/lib/semmle/python/security/performance/ExponentialBackTracking.qll b/python/ql/lib/semmle/python/security/performance/ExponentialBackTracking.qll index 8ba5a75d431..71dff93acb0 100644 --- a/python/ql/lib/semmle/python/security/performance/ExponentialBackTracking.qll +++ b/python/ql/lib/semmle/python/security/performance/ExponentialBackTracking.qll @@ -364,11 +364,5 @@ private predicate isPumpable(State fork, string w) { ) } -/** - * An instantiation of `ReDoSConfiguration` for exponential backtracking. - */ -class ExponentialReDoSConfiguration extends ReDoSConfiguration { - ExponentialReDoSConfiguration() { this = "ExponentialReDoSConfiguration" } - - override predicate isReDoSCandidate(State state, string pump) { isPumpable(state, pump) } -} +/** Holds if `state` has exponential ReDoS */ +predicate hasReDoSResult = ReDoSPruning::hasReDoSResult/4; diff --git a/python/ql/lib/semmle/python/security/performance/ReDoSUtil.qll b/python/ql/lib/semmle/python/security/performance/ReDoSUtil.qll index 8315d2b8225..35f8ffc0e65 100644 --- a/python/ql/lib/semmle/python/security/performance/ReDoSUtil.qll +++ b/python/ql/lib/semmle/python/security/performance/ReDoSUtil.qll @@ -838,38 +838,32 @@ predicate isStartState(State state) { } /** - * A configuration for which parts of a regular expression should be considered relevant for - * the different predicates in `ReDoS.qll`. - * Used to adjust the computations for either superlinear or exponential backtracking. + * Holds if `state` is a candidate for ReDoS with string `pump`. */ -abstract class ReDoSConfiguration extends string { - bindingset[this] - ReDoSConfiguration() { any() } +signature predicate isCandidateSig(State state, string pump); - /** - * Holds if `state` with the pump string `pump` is a candidate for a - * ReDoS vulnerable state. - * This is used to determine which states are considered for the prefix/suffix construction. - */ - abstract predicate isReDoSCandidate(State state, string pump); -} - -module ReDoSPruning { +/** + * A module for pruning candidate ReDoS states. + * The candidates are specified by the `isCandidate` signature predicate. + * The candidates are checked for rejecting suffixes and deduplicated, + * and the resulting ReDoS states are read by the `hasReDoSResult` predicate. + */ +module ReDoSPruning { /** * Holds if repeating `pump` starting at `state` is a candidate for causing backtracking. * No check whether a rejected suffix exists has been made. */ private predicate isReDoSCandidate(State state, string pump) { - any(ReDoSConfiguration conf).isReDoSCandidate(state, pump) and + isCandidate(state, pump) and ( - not any(ReDoSConfiguration conf).isReDoSCandidate(epsilonSucc+(state), _) + not isCandidate(epsilonSucc+(state), _) or epsilonSucc+(state) = state and state = max(State s, Location l | s = epsilonSucc+(state) and l = s.getRepr().getLocation() and - any(ReDoSConfiguration conf).isReDoSCandidate(s, _) and + isCandidate(s, _) and s.getRepr() instanceof InfiniteRepetitionQuantifier | s order by l.getStartLine(), l.getStartColumn(), l.getEndColumn(), l.getEndLine() @@ -887,8 +881,11 @@ module ReDoSPruning { private predicate lastStartState(State state) { exists(RegExpRoot root | state = - max(StateInPumpableRegexp s, Location l | - isStartState(s) and getRoot(s.getRepr()) = root and l = s.getRepr().getLocation() + max(State s, Location l | + s = stateInPumpableRegexp() and + isStartState(s) and + getRoot(s.getRepr()) = root and + l = s.getRepr().getLocation() | s order by @@ -957,14 +954,10 @@ module ReDoSPruning { min(string c | delta(prev, any(InputSymbol symbol | c = intersect(Any(), symbol)), next)) } - /** - * A state within a regular expression that has a pumpable state. - */ - class StateInPumpableRegexp extends State { - pragma[noinline] - StateInPumpableRegexp() { - exists(State s | isReDoSCandidate(s, _) | getRoot(s.getRepr()) = getRoot(this.getRepr())) - } + /** Gets a state within a regular expression that has a pumpable state. */ + pragma[noinline] + State stateInPumpableRegexp() { + exists(State s | isReDoSCandidate(s, _) | getRoot(s.getRepr()) = getRoot(result.getRepr())) } } @@ -1002,26 +995,32 @@ module ReDoSPruning { * This predicate might find impossible suffixes when searching for suffixes of length > 1, which can cause FPs. */ pragma[noinline] - private predicate isLikelyRejectable(StateInPumpableRegexp s) { - // exists a reject edge with some char. - hasRejectEdge(s) - or - hasEdgeToLikelyRejectable(s) - or - // stopping here is rejection - isRejectState(s) + private predicate isLikelyRejectable(State s) { + s = stateInPumpableRegexp() and + ( + // exists a reject edge with some char. + hasRejectEdge(s) + or + hasEdgeToLikelyRejectable(s) + or + // stopping here is rejection + isRejectState(s) + ) } /** * Holds if `s` is not an accept state, and there is no epsilon transition to an accept state. */ - predicate isRejectState(StateInPumpableRegexp s) { not epsilonSucc*(s) = Accept(_) } + predicate isRejectState(State s) { + s = stateInPumpableRegexp() and not epsilonSucc*(s) = Accept(_) + } /** * Holds if there is likely a non-empty suffix leading to rejection starting in `s`. */ pragma[noopt] - predicate hasEdgeToLikelyRejectable(StateInPumpableRegexp s) { + predicate hasEdgeToLikelyRejectable(State s) { + s = stateInPumpableRegexp() and // all edges (at least one) with some char leads to another state that is rejectable. // the `next` states might not share a common suffix, which can cause FPs. exists(string char | char = hasEdgeToLikelyRejectableHelper(s) | @@ -1036,7 +1035,8 @@ module ReDoSPruning { * and `s` has not been found to be rejectable by `hasRejectEdge` or `isRejectState`. */ pragma[noinline] - private string hasEdgeToLikelyRejectableHelper(StateInPumpableRegexp s) { + private string hasEdgeToLikelyRejectableHelper(State s) { + s = stateInPumpableRegexp() and not hasRejectEdge(s) and not isRejectState(s) and deltaClosedChar(s, result, _) @@ -1047,7 +1047,9 @@ module ReDoSPruning { * along epsilon edges, such that there is a transition from * `prev` to `next` that the character symbol `char`. */ - predicate deltaClosedChar(StateInPumpableRegexp prev, string char, StateInPumpableRegexp next) { + predicate deltaClosedChar(State prev, string char, State next) { + prev = stateInPumpableRegexp() and + next = stateInPumpableRegexp() and deltaClosed(prev, getAnInputSymbolMatchingRelevant(char), next) } @@ -1148,7 +1150,7 @@ module ReDoSPruning { exists(int i, string c | s = Match(term, i) | c = min(string w | - any(ReDoSConfiguration conf).isReDoSCandidate(s, w) and + isCandidate(s, w) and SuffixConstruction::reachesOnlyRejectableSuffixes(s, w) | w order by w.length(), w @@ -1173,28 +1175,28 @@ module ReDoSPruning { not exists(PrefixConstruction::prefix(s)) and prefixMsg = "" ) } -} -/** - * Gets the result of backslash-escaping newlines, carriage-returns and - * backslashes in `s`. - */ -bindingset[s] -private string escape(string s) { - result = - s.replaceAll("\\", "\\\\") - .replaceAll("\n", "\\n") - .replaceAll("\r", "\\r") - .replaceAll("\t", "\\t") -} + /** + * Gets the result of backslash-escaping newlines, carriage-returns and + * backslashes in `s`. + */ + bindingset[s] + private string escape(string s) { + result = + s.replaceAll("\\", "\\\\") + .replaceAll("\n", "\\n") + .replaceAll("\r", "\\r") + .replaceAll("\t", "\\t") + } -/** - * Gets `str` with the last `i` characters moved to the front. - * - * We use this to adjust the pump string to match with the beginning of - * a RegExpTerm, so it doesn't start in the middle of a constant. - */ -bindingset[str, i] -private string rotate(string str, int i) { - result = str.suffix(str.length() - i) + str.prefix(str.length() - i) + /** + * Gets `str` with the last `i` characters moved to the front. + * + * We use this to adjust the pump string to match with the beginning of + * a RegExpTerm, so it doesn't start in the middle of a constant. + */ + bindingset[str, i] + private string rotate(string str, int i) { + result = str.suffix(str.length() - i) + str.prefix(str.length() - i) + } } diff --git a/python/ql/lib/semmle/python/security/performance/SuperlinearBackTracking.qll b/python/ql/lib/semmle/python/security/performance/SuperlinearBackTracking.qll index 05edbfd9777..b1916cfd76b 100644 --- a/python/ql/lib/semmle/python/security/performance/SuperlinearBackTracking.qll +++ b/python/ql/lib/semmle/python/security/performance/SuperlinearBackTracking.qll @@ -42,15 +42,6 @@ import ReDoSUtil * It also doesn't find all transitions in the product automaton, which can cause false negatives. */ -/** - * An instantiaion of `ReDoSConfiguration` for superlinear ReDoS. - */ -class SuperLinearReDoSConfiguration extends ReDoSConfiguration { - SuperLinearReDoSConfiguration() { this = "SuperLinearReDoSConfiguration" } - - override predicate isReDoSCandidate(State state, string pump) { isPumpable(_, state, pump) } -} - /** * Gets any root (start) state of a regular expression. */ @@ -386,12 +377,17 @@ predicate isPumpable(State pivot, State succ, string pump) { ) } +/** + * Holds if states starting in `state` can have polynomial backtracking with the string `pump`. + */ +predicate isReDoSCandidate(State state, string pump) { isPumpable(_, state, pump) } + /** * Holds if repetitions of `pump` at `t` will cause polynomial backtracking. */ predicate polynimalReDoS(RegExpTerm t, string pump, string prefixMsg, RegExpTerm prev) { exists(State s, State pivot | - ReDoSPruning::hasReDoSResult(t, pump, s, prefixMsg) and + ReDoSPruning::hasReDoSResult(t, pump, s, prefixMsg) and isPumpable(pivot, s, _) and prev = pivot.getRepr() ) diff --git a/python/ql/src/Security/CWE-730/ReDoS.ql b/python/ql/src/Security/CWE-730/ReDoS.ql index 03b39686f1c..c6d5397e771 100644 --- a/python/ql/src/Security/CWE-730/ReDoS.ql +++ b/python/ql/src/Security/CWE-730/ReDoS.ql @@ -18,7 +18,7 @@ import semmle.python.security.performance.ExponentialBackTracking from RegExpTerm t, string pump, State s, string prefixMsg where - ReDoSPruning::hasReDoSResult(t, pump, s, prefixMsg) and + hasReDoSResult(t, pump, s, prefixMsg) and // exclude verbose mode regexes for now not t.getRegex().getAMode() = "VERBOSE" select t, diff --git a/ruby/ql/lib/codeql/ruby/security/performance/ExponentialBackTracking.qll b/ruby/ql/lib/codeql/ruby/security/performance/ExponentialBackTracking.qll index 8ba5a75d431..71dff93acb0 100644 --- a/ruby/ql/lib/codeql/ruby/security/performance/ExponentialBackTracking.qll +++ b/ruby/ql/lib/codeql/ruby/security/performance/ExponentialBackTracking.qll @@ -364,11 +364,5 @@ private predicate isPumpable(State fork, string w) { ) } -/** - * An instantiation of `ReDoSConfiguration` for exponential backtracking. - */ -class ExponentialReDoSConfiguration extends ReDoSConfiguration { - ExponentialReDoSConfiguration() { this = "ExponentialReDoSConfiguration" } - - override predicate isReDoSCandidate(State state, string pump) { isPumpable(state, pump) } -} +/** Holds if `state` has exponential ReDoS */ +predicate hasReDoSResult = ReDoSPruning::hasReDoSResult/4; diff --git a/ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtil.qll b/ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtil.qll index 8315d2b8225..35f8ffc0e65 100644 --- a/ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtil.qll +++ b/ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtil.qll @@ -838,38 +838,32 @@ predicate isStartState(State state) { } /** - * A configuration for which parts of a regular expression should be considered relevant for - * the different predicates in `ReDoS.qll`. - * Used to adjust the computations for either superlinear or exponential backtracking. + * Holds if `state` is a candidate for ReDoS with string `pump`. */ -abstract class ReDoSConfiguration extends string { - bindingset[this] - ReDoSConfiguration() { any() } +signature predicate isCandidateSig(State state, string pump); - /** - * Holds if `state` with the pump string `pump` is a candidate for a - * ReDoS vulnerable state. - * This is used to determine which states are considered for the prefix/suffix construction. - */ - abstract predicate isReDoSCandidate(State state, string pump); -} - -module ReDoSPruning { +/** + * A module for pruning candidate ReDoS states. + * The candidates are specified by the `isCandidate` signature predicate. + * The candidates are checked for rejecting suffixes and deduplicated, + * and the resulting ReDoS states are read by the `hasReDoSResult` predicate. + */ +module ReDoSPruning { /** * Holds if repeating `pump` starting at `state` is a candidate for causing backtracking. * No check whether a rejected suffix exists has been made. */ private predicate isReDoSCandidate(State state, string pump) { - any(ReDoSConfiguration conf).isReDoSCandidate(state, pump) and + isCandidate(state, pump) and ( - not any(ReDoSConfiguration conf).isReDoSCandidate(epsilonSucc+(state), _) + not isCandidate(epsilonSucc+(state), _) or epsilonSucc+(state) = state and state = max(State s, Location l | s = epsilonSucc+(state) and l = s.getRepr().getLocation() and - any(ReDoSConfiguration conf).isReDoSCandidate(s, _) and + isCandidate(s, _) and s.getRepr() instanceof InfiniteRepetitionQuantifier | s order by l.getStartLine(), l.getStartColumn(), l.getEndColumn(), l.getEndLine() @@ -887,8 +881,11 @@ module ReDoSPruning { private predicate lastStartState(State state) { exists(RegExpRoot root | state = - max(StateInPumpableRegexp s, Location l | - isStartState(s) and getRoot(s.getRepr()) = root and l = s.getRepr().getLocation() + max(State s, Location l | + s = stateInPumpableRegexp() and + isStartState(s) and + getRoot(s.getRepr()) = root and + l = s.getRepr().getLocation() | s order by @@ -957,14 +954,10 @@ module ReDoSPruning { min(string c | delta(prev, any(InputSymbol symbol | c = intersect(Any(), symbol)), next)) } - /** - * A state within a regular expression that has a pumpable state. - */ - class StateInPumpableRegexp extends State { - pragma[noinline] - StateInPumpableRegexp() { - exists(State s | isReDoSCandidate(s, _) | getRoot(s.getRepr()) = getRoot(this.getRepr())) - } + /** Gets a state within a regular expression that has a pumpable state. */ + pragma[noinline] + State stateInPumpableRegexp() { + exists(State s | isReDoSCandidate(s, _) | getRoot(s.getRepr()) = getRoot(result.getRepr())) } } @@ -1002,26 +995,32 @@ module ReDoSPruning { * This predicate might find impossible suffixes when searching for suffixes of length > 1, which can cause FPs. */ pragma[noinline] - private predicate isLikelyRejectable(StateInPumpableRegexp s) { - // exists a reject edge with some char. - hasRejectEdge(s) - or - hasEdgeToLikelyRejectable(s) - or - // stopping here is rejection - isRejectState(s) + private predicate isLikelyRejectable(State s) { + s = stateInPumpableRegexp() and + ( + // exists a reject edge with some char. + hasRejectEdge(s) + or + hasEdgeToLikelyRejectable(s) + or + // stopping here is rejection + isRejectState(s) + ) } /** * Holds if `s` is not an accept state, and there is no epsilon transition to an accept state. */ - predicate isRejectState(StateInPumpableRegexp s) { not epsilonSucc*(s) = Accept(_) } + predicate isRejectState(State s) { + s = stateInPumpableRegexp() and not epsilonSucc*(s) = Accept(_) + } /** * Holds if there is likely a non-empty suffix leading to rejection starting in `s`. */ pragma[noopt] - predicate hasEdgeToLikelyRejectable(StateInPumpableRegexp s) { + predicate hasEdgeToLikelyRejectable(State s) { + s = stateInPumpableRegexp() and // all edges (at least one) with some char leads to another state that is rejectable. // the `next` states might not share a common suffix, which can cause FPs. exists(string char | char = hasEdgeToLikelyRejectableHelper(s) | @@ -1036,7 +1035,8 @@ module ReDoSPruning { * and `s` has not been found to be rejectable by `hasRejectEdge` or `isRejectState`. */ pragma[noinline] - private string hasEdgeToLikelyRejectableHelper(StateInPumpableRegexp s) { + private string hasEdgeToLikelyRejectableHelper(State s) { + s = stateInPumpableRegexp() and not hasRejectEdge(s) and not isRejectState(s) and deltaClosedChar(s, result, _) @@ -1047,7 +1047,9 @@ module ReDoSPruning { * along epsilon edges, such that there is a transition from * `prev` to `next` that the character symbol `char`. */ - predicate deltaClosedChar(StateInPumpableRegexp prev, string char, StateInPumpableRegexp next) { + predicate deltaClosedChar(State prev, string char, State next) { + prev = stateInPumpableRegexp() and + next = stateInPumpableRegexp() and deltaClosed(prev, getAnInputSymbolMatchingRelevant(char), next) } @@ -1148,7 +1150,7 @@ module ReDoSPruning { exists(int i, string c | s = Match(term, i) | c = min(string w | - any(ReDoSConfiguration conf).isReDoSCandidate(s, w) and + isCandidate(s, w) and SuffixConstruction::reachesOnlyRejectableSuffixes(s, w) | w order by w.length(), w @@ -1173,28 +1175,28 @@ module ReDoSPruning { not exists(PrefixConstruction::prefix(s)) and prefixMsg = "" ) } -} -/** - * Gets the result of backslash-escaping newlines, carriage-returns and - * backslashes in `s`. - */ -bindingset[s] -private string escape(string s) { - result = - s.replaceAll("\\", "\\\\") - .replaceAll("\n", "\\n") - .replaceAll("\r", "\\r") - .replaceAll("\t", "\\t") -} + /** + * Gets the result of backslash-escaping newlines, carriage-returns and + * backslashes in `s`. + */ + bindingset[s] + private string escape(string s) { + result = + s.replaceAll("\\", "\\\\") + .replaceAll("\n", "\\n") + .replaceAll("\r", "\\r") + .replaceAll("\t", "\\t") + } -/** - * Gets `str` with the last `i` characters moved to the front. - * - * We use this to adjust the pump string to match with the beginning of - * a RegExpTerm, so it doesn't start in the middle of a constant. - */ -bindingset[str, i] -private string rotate(string str, int i) { - result = str.suffix(str.length() - i) + str.prefix(str.length() - i) + /** + * Gets `str` with the last `i` characters moved to the front. + * + * We use this to adjust the pump string to match with the beginning of + * a RegExpTerm, so it doesn't start in the middle of a constant. + */ + bindingset[str, i] + private string rotate(string str, int i) { + result = str.suffix(str.length() - i) + str.prefix(str.length() - i) + } } diff --git a/ruby/ql/lib/codeql/ruby/security/performance/SuperlinearBackTracking.qll b/ruby/ql/lib/codeql/ruby/security/performance/SuperlinearBackTracking.qll index 05edbfd9777..b1916cfd76b 100644 --- a/ruby/ql/lib/codeql/ruby/security/performance/SuperlinearBackTracking.qll +++ b/ruby/ql/lib/codeql/ruby/security/performance/SuperlinearBackTracking.qll @@ -42,15 +42,6 @@ import ReDoSUtil * It also doesn't find all transitions in the product automaton, which can cause false negatives. */ -/** - * An instantiaion of `ReDoSConfiguration` for superlinear ReDoS. - */ -class SuperLinearReDoSConfiguration extends ReDoSConfiguration { - SuperLinearReDoSConfiguration() { this = "SuperLinearReDoSConfiguration" } - - override predicate isReDoSCandidate(State state, string pump) { isPumpable(_, state, pump) } -} - /** * Gets any root (start) state of a regular expression. */ @@ -386,12 +377,17 @@ predicate isPumpable(State pivot, State succ, string pump) { ) } +/** + * Holds if states starting in `state` can have polynomial backtracking with the string `pump`. + */ +predicate isReDoSCandidate(State state, string pump) { isPumpable(_, state, pump) } + /** * Holds if repetitions of `pump` at `t` will cause polynomial backtracking. */ predicate polynimalReDoS(RegExpTerm t, string pump, string prefixMsg, RegExpTerm prev) { exists(State s, State pivot | - ReDoSPruning::hasReDoSResult(t, pump, s, prefixMsg) and + ReDoSPruning::hasReDoSResult(t, pump, s, prefixMsg) and isPumpable(pivot, s, _) and prev = pivot.getRepr() ) diff --git a/ruby/ql/src/queries/security/cwe-1333/ReDoS.ql b/ruby/ql/src/queries/security/cwe-1333/ReDoS.ql index 4491b79da2a..e9ccce7fd04 100644 --- a/ruby/ql/src/queries/security/cwe-1333/ReDoS.ql +++ b/ruby/ql/src/queries/security/cwe-1333/ReDoS.ql @@ -19,7 +19,7 @@ import codeql.ruby.security.performance.ReDoSUtil import codeql.ruby.Regexp from RegExpTerm t, string pump, State s, string prefixMsg -where ReDoSPruning::hasReDoSResult(t, pump, s, prefixMsg) +where hasReDoSResult(t, pump, s, prefixMsg) select t, "This part of the regular expression may cause exponential backtracking on strings " + prefixMsg + "containing many repetitions of '" + pump + "'." From 7fb3d81d2fb0489bb89efa133e348ff88f0ecba0 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Mon, 14 Feb 2022 18:49:57 +0100 Subject: [PATCH 003/600] add further normalization of char classses --- .../java/security/performance/ReDoSUtil.qll | 66 ++++- .../security/performance/ReDoSUtil.qll | 66 ++++- .../ReDoS/PolynomialBackTracking.expected | 267 +++++++++--------- .../ReDoS/PolynomialReDoS.expected | 28 +- .../Performance/ReDoS/ReDoS.expected | 82 +++--- .../python/security/performance/ReDoSUtil.qll | 66 ++++- .../Security/CWE-730-ReDoS/ReDoS.expected | 14 +- .../ruby/security/performance/ReDoSUtil.qll | 66 ++++- .../cwe-1333-exponential-redos/ReDoS.expected | 12 +- 9 files changed, 415 insertions(+), 252 deletions(-) diff --git a/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll b/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll index 35f8ffc0e65..58900594158 100644 --- a/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll +++ b/java/ql/lib/semmle/code/java/security/performance/ReDoSUtil.qll @@ -199,7 +199,7 @@ CharClass getCanonicalCharClass(RegExpTerm term) { /** * Holds if `a` and `b` are input symbols from the same regexp. */ -private predicate sharesRoot(TInputSymbol a, TInputSymbol b) { +private predicate sharesRoot(InputSymbol a, InputSymbol b) { exists(RegExpRoot root | belongsTo(a, root) and belongsTo(b, root) @@ -209,7 +209,7 @@ private predicate sharesRoot(TInputSymbol a, TInputSymbol b) { /** * Holds if the `a` is an input symbol from a regexp that has root `root`. */ -private predicate belongsTo(TInputSymbol a, RegExpRoot root) { +private predicate belongsTo(InputSymbol a, RegExpRoot root) { exists(State s | getRoot(s.getRepr()) = root | delta(s, a, _) or @@ -378,6 +378,13 @@ private module CharacterClasses { ) } + bindingset[char, cc] + private string caseNormalize(string char, RegExpTerm cc) { + if RegExpFlags::isIgnoreCase(cc.getRootTerm()) + then result = char.toLowerCase() + else result = char + } + /** * An implementation of `CharacterClass` for positive (non inverted) character classes. */ @@ -386,7 +393,7 @@ private module CharacterClasses { PositiveCharacterClass() { this = getCanonicalCharClass(cc) and not cc.isInverted() } - override string getARelevantChar() { result = getAMentionedChar(cc) } + override string getARelevantChar() { result = caseNormalize(getAMentionedChar(cc), cc) } override predicate matches(string char) { hasChildThatMatches(cc, char) } } @@ -400,8 +407,8 @@ private module CharacterClasses { InvertedCharacterClass() { this = getCanonicalCharClass(cc) and cc.isInverted() } override string getARelevantChar() { - result = nextChar(getAMentionedChar(cc)) or - nextChar(result) = getAMentionedChar(cc) + result = nextChar(caseNormalize(getAMentionedChar(cc), cc)) or + nextChar(result) = caseNormalize(getAMentionedChar(cc), cc) } bindingset[char] @@ -428,13 +435,12 @@ private module CharacterClasses { */ private class PositiveCharacterClassEscape extends CharacterClass { string charClass; + RegExpTerm cc; PositiveCharacterClassEscape() { - exists(RegExpTerm cc | - isEscapeClass(cc, charClass) and - this = getCanonicalCharClass(cc) and - charClass = ["d", "s", "w"] - ) + isEscapeClass(cc, charClass) and + this = getCanonicalCharClass(cc) and + charClass = ["d", "s", "w"] } override string getARelevantChar() { @@ -445,7 +451,9 @@ private module CharacterClasses { result = " " or charClass = "w" and - result = ["a", "Z", "_", "0", "9"] + if RegExpFlags::isIgnoreCase(cc.getRootTerm()) + then result = ["a", "z", "_", "0", "9"] + else result = ["a", "Z", "_", "0", "9"] } override predicate matches(string char) { classEscapeMatches(charClass, char) } @@ -492,6 +500,34 @@ private module CharacterClasses { not classEscapeMatches(charClass.toLowerCase(), char) } } + + /** Gets a representative for all char classes that match the same chars as `c`. */ + CharacterClass normalize(CharacterClass c) { + exists(string normalization | + normalization = getMormalizationString(c) and + result = + min(CharacterClass cc, string raw | + getMormalizationString(cc) = normalization and cc = CharClass(raw) + | + cc order by raw + ) + ) + } + + /** Gets a string representing all the chars matched by `c` */ + private string getMormalizationString(CharacterClass c) { + (c instanceof PositiveCharacterClass or c instanceof PositiveCharacterClassEscape) and + result = concat(string char | c.matches(char) and char = CharacterClasses::getARelevantChar()) + or + (c instanceof InvertedCharacterClass or c instanceof NegativeCharacterClassEscape) and + // the string produced by the concat can not contain repeated chars + // so by starting the below with "nn" we can guarantee that + // it will not overlap with the above case. + // and a negative char class can never match the same chars as a positive one, so we don't miss any results from this. + result = + "nn:" + + concat(string char | not c.matches(char) and char = CharacterClasses::getARelevantChar()) + } } private class EdgeLabel extends TInputSymbol { @@ -620,13 +656,17 @@ predicate delta(State q1, EdgeLabel lbl, State q2) { cc.isUniversalClass() and q1 = before(cc) and lbl = Any() and q2 = after(cc) or q1 = before(cc) and - lbl = CharClass(cc.getRawValue() + "|" + getCanonicalizationFlags(cc.getRootTerm())) and + lbl = + CharacterClasses::normalize(CharClass(cc.getRawValue() + "|" + + getCanonicalizationFlags(cc.getRootTerm()))) and q2 = after(cc) ) or exists(RegExpTerm cc | isEscapeClass(cc, _) | q1 = before(cc) and - lbl = CharClass(cc.getRawValue() + "|" + getCanonicalizationFlags(cc.getRootTerm())) and + lbl = + CharacterClasses::normalize(CharClass(cc.getRawValue() + "|" + + getCanonicalizationFlags(cc.getRootTerm()))) and q2 = after(cc) ) or diff --git a/javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtil.qll b/javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtil.qll index 35f8ffc0e65..58900594158 100644 --- a/javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtil.qll +++ b/javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtil.qll @@ -199,7 +199,7 @@ CharClass getCanonicalCharClass(RegExpTerm term) { /** * Holds if `a` and `b` are input symbols from the same regexp. */ -private predicate sharesRoot(TInputSymbol a, TInputSymbol b) { +private predicate sharesRoot(InputSymbol a, InputSymbol b) { exists(RegExpRoot root | belongsTo(a, root) and belongsTo(b, root) @@ -209,7 +209,7 @@ private predicate sharesRoot(TInputSymbol a, TInputSymbol b) { /** * Holds if the `a` is an input symbol from a regexp that has root `root`. */ -private predicate belongsTo(TInputSymbol a, RegExpRoot root) { +private predicate belongsTo(InputSymbol a, RegExpRoot root) { exists(State s | getRoot(s.getRepr()) = root | delta(s, a, _) or @@ -378,6 +378,13 @@ private module CharacterClasses { ) } + bindingset[char, cc] + private string caseNormalize(string char, RegExpTerm cc) { + if RegExpFlags::isIgnoreCase(cc.getRootTerm()) + then result = char.toLowerCase() + else result = char + } + /** * An implementation of `CharacterClass` for positive (non inverted) character classes. */ @@ -386,7 +393,7 @@ private module CharacterClasses { PositiveCharacterClass() { this = getCanonicalCharClass(cc) and not cc.isInverted() } - override string getARelevantChar() { result = getAMentionedChar(cc) } + override string getARelevantChar() { result = caseNormalize(getAMentionedChar(cc), cc) } override predicate matches(string char) { hasChildThatMatches(cc, char) } } @@ -400,8 +407,8 @@ private module CharacterClasses { InvertedCharacterClass() { this = getCanonicalCharClass(cc) and cc.isInverted() } override string getARelevantChar() { - result = nextChar(getAMentionedChar(cc)) or - nextChar(result) = getAMentionedChar(cc) + result = nextChar(caseNormalize(getAMentionedChar(cc), cc)) or + nextChar(result) = caseNormalize(getAMentionedChar(cc), cc) } bindingset[char] @@ -428,13 +435,12 @@ private module CharacterClasses { */ private class PositiveCharacterClassEscape extends CharacterClass { string charClass; + RegExpTerm cc; PositiveCharacterClassEscape() { - exists(RegExpTerm cc | - isEscapeClass(cc, charClass) and - this = getCanonicalCharClass(cc) and - charClass = ["d", "s", "w"] - ) + isEscapeClass(cc, charClass) and + this = getCanonicalCharClass(cc) and + charClass = ["d", "s", "w"] } override string getARelevantChar() { @@ -445,7 +451,9 @@ private module CharacterClasses { result = " " or charClass = "w" and - result = ["a", "Z", "_", "0", "9"] + if RegExpFlags::isIgnoreCase(cc.getRootTerm()) + then result = ["a", "z", "_", "0", "9"] + else result = ["a", "Z", "_", "0", "9"] } override predicate matches(string char) { classEscapeMatches(charClass, char) } @@ -492,6 +500,34 @@ private module CharacterClasses { not classEscapeMatches(charClass.toLowerCase(), char) } } + + /** Gets a representative for all char classes that match the same chars as `c`. */ + CharacterClass normalize(CharacterClass c) { + exists(string normalization | + normalization = getMormalizationString(c) and + result = + min(CharacterClass cc, string raw | + getMormalizationString(cc) = normalization and cc = CharClass(raw) + | + cc order by raw + ) + ) + } + + /** Gets a string representing all the chars matched by `c` */ + private string getMormalizationString(CharacterClass c) { + (c instanceof PositiveCharacterClass or c instanceof PositiveCharacterClassEscape) and + result = concat(string char | c.matches(char) and char = CharacterClasses::getARelevantChar()) + or + (c instanceof InvertedCharacterClass or c instanceof NegativeCharacterClassEscape) and + // the string produced by the concat can not contain repeated chars + // so by starting the below with "nn" we can guarantee that + // it will not overlap with the above case. + // and a negative char class can never match the same chars as a positive one, so we don't miss any results from this. + result = + "nn:" + + concat(string char | not c.matches(char) and char = CharacterClasses::getARelevantChar()) + } } private class EdgeLabel extends TInputSymbol { @@ -620,13 +656,17 @@ predicate delta(State q1, EdgeLabel lbl, State q2) { cc.isUniversalClass() and q1 = before(cc) and lbl = Any() and q2 = after(cc) or q1 = before(cc) and - lbl = CharClass(cc.getRawValue() + "|" + getCanonicalizationFlags(cc.getRootTerm())) and + lbl = + CharacterClasses::normalize(CharClass(cc.getRawValue() + "|" + + getCanonicalizationFlags(cc.getRootTerm()))) and q2 = after(cc) ) or exists(RegExpTerm cc | isEscapeClass(cc, _) | q1 = before(cc) and - lbl = CharClass(cc.getRawValue() + "|" + getCanonicalizationFlags(cc.getRootTerm())) and + lbl = + CharacterClasses::normalize(CharClass(cc.getRawValue() + "|" + + getCanonicalizationFlags(cc.getRootTerm()))) and q2 = after(cc) ) or diff --git a/javascript/ql/test/query-tests/Performance/ReDoS/PolynomialBackTracking.expected b/javascript/ql/test/query-tests/Performance/ReDoS/PolynomialBackTracking.expected index 96919f8717d..8aac110e9e0 100644 --- a/javascript/ql/test/query-tests/Performance/ReDoS/PolynomialBackTracking.expected +++ b/javascript/ql/test/query-tests/Performance/ReDoS/PolynomialBackTracking.expected @@ -1,5 +1,5 @@ -| highlight.js:2:26:2:979 | ((traffic-flow\|traffic-generator\|firewall\|scheduler\|aaa\|accounting\|address-list\|address\|align\|area\|bandwidth-server\|bfd\|bgp\|bridge\|client\|clock\|community\|config\|connection\|console\|customer\|default\|dhcp-client\|dhcp-server\|discovery\|dns\|e-mail\|ethernet\|filter\|firewall\|firmware\|gps\|graphing\|group\|hardware\|health\|hotspot\|identity\|igmp-proxy\|incoming\|instance\|interface\|ip\|ipsec\|ipv6\|irq\|l2tp-server\|lcd\|ldp\|logging\|mac-server\|mac-winbox\|mangle\|manual\|mirror\|mme\|mpls\|nat\|nd\|neighbor\|network\|note\|ntp\|ospf\|ospf-v3\|ovpn-server\|page\|peer\|pim\|ping\|policy\|pool\|port\|ppp\|pppoe-client\|pptp-server\|prefix\|profile\|proposal\|proxy\|queue\|radius\|resource\|rip\|ripng\|route\|routing\|screen\|script\|security-profiles\|server\|service\|service-port\|settings\|shares\|smb\|sms\|sniffer\|snmp\|snooper\|socks\|sstp-server\|system\|tool\|tracking\|type\|upgrade\|upnp\|user-manager\|users\|user\|vlan\|secret\|vrrp\|watchdog\|web-access\|wireless\|pptp\|pppoe\|lan\|wan\|layer7-protocol\|lease\|simple\|raw);?\\s)+ | Strings starting with '/' and with many repetitions of 'ip ' can start matching anywhere after the start of the preceeding (\\.\\.\\/\|\\/\|\\s)((traffic-flow\|traffic-generator\|firewall\|scheduler\|aaa\|accounting\|address-list\|address\|align\|area\|bandwidth-server\|bfd\|bgp\|bridge\|client\|clock\|community\|config\|connection\|console\|customer\|default\|dhcp-client\|dhcp-server\|discovery\|dns\|e-mail\|ethernet\|filter\|firewall\|firmware\|gps\|graphing\|group\|hardware\|health\|hotspot\|identity\|igmp-proxy\|incoming\|instance\|interface\|ip\|ipsec\|ipv6\|irq\|l2tp-server\|lcd\|ldp\|logging\|mac-server\|mac-winbox\|mangle\|manual\|mirror\|mme\|mpls\|nat\|nd\|neighbor\|network\|note\|ntp\|ospf\|ospf-v3\|ovpn-server\|page\|peer\|pim\|ping\|policy\|pool\|port\|ppp\|pppoe-client\|pptp-server\|prefix\|profile\|proposal\|proxy\|queue\|radius\|resource\|rip\|ripng\|route\|routing\|screen\|script\|security-profiles\|server\|service\|service-port\|settings\|shares\|smb\|sms\|sniffer\|snmp\|snooper\|socks\|sstp-server\|system\|tool\|tracking\|type\|upgrade\|upnp\|user-manager\|users\|user\|vlan\|secret\|vrrp\|watchdog\|web-access\|wireless\|pptp\|pppoe\|lan\|wan\|layer7-protocol\|lease\|simple\|raw);?\\s)+X | -| highlight.js:3:27:3:971 | ((traffic-flow\|traffic-generator\|firewall\|scheduler\|aaa\|accounting\|address-list\|address\|align\|area\|bandwidth-server\|bfd\|bgp\|bridge\|client\|clock\|community\|config\|connection\|console\|customer\|default\|dhcp-client\|dhcp-server\|discovery\|dns\|e-mail\|ethernet\|filter\|firmware\|gps\|graphing\|group\|hardware\|health\|hotspot\|identity\|igmp-proxy\|incoming\|instance\|interface\|ip\|ipsec\|ipv6\|irq\|l2tp-server\|lcd\|ldp\|logging\|mac-server\|mac-winbox\|mangle\|manual\|mirror\|mme\|mpls\|nat\|nd\|neighbor\|network\|note\|ntp\|ospf\|ospf-v3\|ovpn-server\|page\|peer\|pim\|ping\|policy\|pool\|port\|ppp\|pppoe-client\|pptp-server\|prefix\|profile\|proposal\|proxy\|queue\|radius\|resource\|rip\|ripng\|route\|routing\|screen\|script\|security-profiles\|server\|service\|service-port\|settings\|shares\|smb\|sms\|sniffer\|snmp\|snooper\|socks\|sstp-server\|system\|tool\|tracking\|type\|upgrade\|upnp\|user-manager\|users\|user\|vlan\|secret\|vrrp\|watchdog\|web-access\|wireless\|pptp\|pppoe\|lan\|wan\|layer7-protocol\|lease\|simple\|raw);?\\s)+ | Strings starting with '/' and with many repetitions of 'ip ' can start matching anywhere after the start of the preceeding (\\.\\.\\/\|\\/\|\\s)((traffic-flow\|traffic-generator\|firewall\|scheduler\|aaa\|accounting\|address-list\|address\|align\|area\|bandwidth-server\|bfd\|bgp\|bridge\|client\|clock\|community\|config\|connection\|console\|customer\|default\|dhcp-client\|dhcp-server\|discovery\|dns\|e-mail\|ethernet\|filter\|firmware\|gps\|graphing\|group\|hardware\|health\|hotspot\|identity\|igmp-proxy\|incoming\|instance\|interface\|ip\|ipsec\|ipv6\|irq\|l2tp-server\|lcd\|ldp\|logging\|mac-server\|mac-winbox\|mangle\|manual\|mirror\|mme\|mpls\|nat\|nd\|neighbor\|network\|note\|ntp\|ospf\|ospf-v3\|ovpn-server\|page\|peer\|pim\|ping\|policy\|pool\|port\|ppp\|pppoe-client\|pptp-server\|prefix\|profile\|proposal\|proxy\|queue\|radius\|resource\|rip\|ripng\|route\|routing\|screen\|script\|security-profiles\|server\|service\|service-port\|settings\|shares\|smb\|sms\|sniffer\|snmp\|snooper\|socks\|sstp-server\|system\|tool\|tracking\|type\|upgrade\|upnp\|user-manager\|users\|user\|vlan\|secret\|vrrp\|watchdog\|web-access\|wireless\|pptp\|pppoe\|lan\|wan\|layer7-protocol\|lease\|simple\|raw);?\\s)+X | +| highlight.js:2:26:2:979 | ((traffic-flow\|traffic-generator\|firewall\|scheduler\|aaa\|accounting\|address-list\|address\|align\|area\|bandwidth-server\|bfd\|bgp\|bridge\|client\|clock\|community\|config\|connection\|console\|customer\|default\|dhcp-client\|dhcp-server\|discovery\|dns\|e-mail\|ethernet\|filter\|firewall\|firmware\|gps\|graphing\|group\|hardware\|health\|hotspot\|identity\|igmp-proxy\|incoming\|instance\|interface\|ip\|ipsec\|ipv6\|irq\|l2tp-server\|lcd\|ldp\|logging\|mac-server\|mac-winbox\|mangle\|manual\|mirror\|mme\|mpls\|nat\|nd\|neighbor\|network\|note\|ntp\|ospf\|ospf-v3\|ovpn-server\|page\|peer\|pim\|ping\|policy\|pool\|port\|ppp\|pppoe-client\|pptp-server\|prefix\|profile\|proposal\|proxy\|queue\|radius\|resource\|rip\|ripng\|route\|routing\|screen\|script\|security-profiles\|server\|service\|service-port\|settings\|shares\|smb\|sms\|sniffer\|snmp\|snooper\|socks\|sstp-server\|system\|tool\|tracking\|type\|upgrade\|upnp\|user-manager\|users\|user\|vlan\|secret\|vrrp\|watchdog\|web-access\|wireless\|pptp\|pppoe\|lan\|wan\|layer7-protocol\|lease\|simple\|raw);?\\s)+ | Strings starting with '/' and with many repetitions of 'ip\\t' can start matching anywhere after the start of the preceeding (\\.\\.\\/\|\\/\|\\s)((traffic-flow\|traffic-generator\|firewall\|scheduler\|aaa\|accounting\|address-list\|address\|align\|area\|bandwidth-server\|bfd\|bgp\|bridge\|client\|clock\|community\|config\|connection\|console\|customer\|default\|dhcp-client\|dhcp-server\|discovery\|dns\|e-mail\|ethernet\|filter\|firewall\|firmware\|gps\|graphing\|group\|hardware\|health\|hotspot\|identity\|igmp-proxy\|incoming\|instance\|interface\|ip\|ipsec\|ipv6\|irq\|l2tp-server\|lcd\|ldp\|logging\|mac-server\|mac-winbox\|mangle\|manual\|mirror\|mme\|mpls\|nat\|nd\|neighbor\|network\|note\|ntp\|ospf\|ospf-v3\|ovpn-server\|page\|peer\|pim\|ping\|policy\|pool\|port\|ppp\|pppoe-client\|pptp-server\|prefix\|profile\|proposal\|proxy\|queue\|radius\|resource\|rip\|ripng\|route\|routing\|screen\|script\|security-profiles\|server\|service\|service-port\|settings\|shares\|smb\|sms\|sniffer\|snmp\|snooper\|socks\|sstp-server\|system\|tool\|tracking\|type\|upgrade\|upnp\|user-manager\|users\|user\|vlan\|secret\|vrrp\|watchdog\|web-access\|wireless\|pptp\|pppoe\|lan\|wan\|layer7-protocol\|lease\|simple\|raw);?\\s)+X | +| highlight.js:3:27:3:971 | ((traffic-flow\|traffic-generator\|firewall\|scheduler\|aaa\|accounting\|address-list\|address\|align\|area\|bandwidth-server\|bfd\|bgp\|bridge\|client\|clock\|community\|config\|connection\|console\|customer\|default\|dhcp-client\|dhcp-server\|discovery\|dns\|e-mail\|ethernet\|filter\|firmware\|gps\|graphing\|group\|hardware\|health\|hotspot\|identity\|igmp-proxy\|incoming\|instance\|interface\|ip\|ipsec\|ipv6\|irq\|l2tp-server\|lcd\|ldp\|logging\|mac-server\|mac-winbox\|mangle\|manual\|mirror\|mme\|mpls\|nat\|nd\|neighbor\|network\|note\|ntp\|ospf\|ospf-v3\|ovpn-server\|page\|peer\|pim\|ping\|policy\|pool\|port\|ppp\|pppoe-client\|pptp-server\|prefix\|profile\|proposal\|proxy\|queue\|radius\|resource\|rip\|ripng\|route\|routing\|screen\|script\|security-profiles\|server\|service\|service-port\|settings\|shares\|smb\|sms\|sniffer\|snmp\|snooper\|socks\|sstp-server\|system\|tool\|tracking\|type\|upgrade\|upnp\|user-manager\|users\|user\|vlan\|secret\|vrrp\|watchdog\|web-access\|wireless\|pptp\|pppoe\|lan\|wan\|layer7-protocol\|lease\|simple\|raw);?\\s)+ | Strings starting with '/' and with many repetitions of 'ip\\t' can start matching anywhere after the start of the preceeding (\\.\\.\\/\|\\/\|\\s)((traffic-flow\|traffic-generator\|firewall\|scheduler\|aaa\|accounting\|address-list\|address\|align\|area\|bandwidth-server\|bfd\|bgp\|bridge\|client\|clock\|community\|config\|connection\|console\|customer\|default\|dhcp-client\|dhcp-server\|discovery\|dns\|e-mail\|ethernet\|filter\|firmware\|gps\|graphing\|group\|hardware\|health\|hotspot\|identity\|igmp-proxy\|incoming\|instance\|interface\|ip\|ipsec\|ipv6\|irq\|l2tp-server\|lcd\|ldp\|logging\|mac-server\|mac-winbox\|mangle\|manual\|mirror\|mme\|mpls\|nat\|nd\|neighbor\|network\|note\|ntp\|ospf\|ospf-v3\|ovpn-server\|page\|peer\|pim\|ping\|policy\|pool\|port\|ppp\|pppoe-client\|pptp-server\|prefix\|profile\|proposal\|proxy\|queue\|radius\|resource\|rip\|ripng\|route\|routing\|screen\|script\|security-profiles\|server\|service\|service-port\|settings\|shares\|smb\|sms\|sniffer\|snmp\|snooper\|socks\|sstp-server\|system\|tool\|tracking\|type\|upgrade\|upnp\|user-manager\|users\|user\|vlan\|secret\|vrrp\|watchdog\|web-access\|wireless\|pptp\|pppoe\|lan\|wan\|layer7-protocol\|lease\|simple\|raw);?\\s)+X | | highlight.js:6:12:6:695 | (Add\|Clear\|Close\|Copy\|Enter\|Exit\|Find\|Format\|Get\|Hide\|Join\|Lock\|Move\|New\|Open\|Optimize\|Pop\|Push\|Redo\|Remove\|Rename\|Reset\|Resize\|Search\|Select\|Set\|Show\|Skip\|Split\|Step\|Switch\|Undo\|Unlock\|Watch\|Backup\|Checkpoint\|Compare\|Compress\|Convert\|ConvertFrom\|ConvertTo\|Dismount\|Edit\|Expand\|Export\|Group\|Import\|Initialize\|Limit\|Merge\|New\|Out\|Publish\|Restore\|Save\|Sync\|Unpublish\|Update\|Approve\|Assert\|Complete\|Confirm\|Deny\|Disable\|Enable\|Install\|Invoke\|Register\|Request\|Restart\|Resume\|Start\|Stop\|Submit\|Suspend\|Uninstall\|Unregister\|Wait\|Debug\|Measure\|Ping\|Repair\|Resolve\|Test\|Trace\|Connect\|Disconnect\|Read\|Receive\|Send\|Write\|Block\|Grant\|Protect\|Revoke\|Unblock\|Unprotect\|Use\|ForEach\|Sort\|Tee\|Where)+ | Strings with many repetitions of 'Add' can start matching anywhere after the start of the preceeding (Add\|Clear\|Close\|Copy\|Enter\|Exit\|Find\|Format\|Get\|Hide\|Join\|Lock\|Move\|New\|Open\|Optimize\|Pop\|Push\|Redo\|Remove\|Rename\|Reset\|Resize\|Search\|Select\|Set\|Show\|Skip\|Split\|Step\|Switch\|Undo\|Unlock\|Watch\|Backup\|Checkpoint\|Compare\|Compress\|Convert\|ConvertFrom\|ConvertTo\|Dismount\|Edit\|Expand\|Export\|Group\|Import\|Initialize\|Limit\|Merge\|New\|Out\|Publish\|Restore\|Save\|Sync\|Unpublish\|Update\|Approve\|Assert\|Complete\|Confirm\|Deny\|Disable\|Enable\|Install\|Invoke\|Register\|Request\|Restart\|Resume\|Start\|Stop\|Submit\|Suspend\|Uninstall\|Unregister\|Wait\|Debug\|Measure\|Ping\|Repair\|Resolve\|Test\|Trace\|Connect\|Disconnect\|Read\|Receive\|Send\|Write\|Block\|Grant\|Protect\|Revoke\|Unblock\|Unprotect\|Use\|ForEach\|Sort\|Tee\|Where)+(-)[\\w\\d]+ | | highlight.js:7:13:7:692 | (Add\|Clear\|Close\|Copy\|Enter\|Exit\|Find\|Format\|Get\|Hide\|Join\|Lock\|Move\|New\|Open\|Optimize\|Pop\|Push\|Redo\|Remove\|Rename\|Reset\|Resize\|Search\|Select\|Set\|Show\|Skip\|Split\|Step\|Switch\|Undo\|Unlock\|Watch\|Backup\|Checkpoint\|Compare\|Compress\|Convert\|ConvertFrom\|ConvertTo\|Dismount\|Edit\|Expand\|Export\|Group\|Import\|Initialize\|Limit\|Merge\|Out\|Publish\|Restore\|Save\|Sync\|Unpublish\|Update\|Approve\|Assert\|Complete\|Confirm\|Deny\|Disable\|Enable\|Install\|Invoke\|Register\|Request\|Restart\|Resume\|Start\|Stop\|Submit\|Suspend\|Uninstall\|Unregister\|Wait\|Debug\|Measure\|Ping\|Repair\|Resolve\|Test\|Trace\|Connect\|Disconnect\|Read\|Receive\|Send\|Write\|Block\|Grant\|Protect\|Revoke\|Unblock\|Unprotect\|Use\|ForEach\|Sort\|Tee\|Where)+ | Strings with many repetitions of 'Add' can start matching anywhere after the start of the preceeding (Add\|Clear\|Close\|Copy\|Enter\|Exit\|Find\|Format\|Get\|Hide\|Join\|Lock\|Move\|New\|Open\|Optimize\|Pop\|Push\|Redo\|Remove\|Rename\|Reset\|Resize\|Search\|Select\|Set\|Show\|Skip\|Split\|Step\|Switch\|Undo\|Unlock\|Watch\|Backup\|Checkpoint\|Compare\|Compress\|Convert\|ConvertFrom\|ConvertTo\|Dismount\|Edit\|Expand\|Export\|Group\|Import\|Initialize\|Limit\|Merge\|Out\|Publish\|Restore\|Save\|Sync\|Unpublish\|Update\|Approve\|Assert\|Complete\|Confirm\|Deny\|Disable\|Enable\|Install\|Invoke\|Register\|Request\|Restart\|Resume\|Start\|Stop\|Submit\|Suspend\|Uninstall\|Unregister\|Wait\|Debug\|Measure\|Ping\|Repair\|Resolve\|Test\|Trace\|Connect\|Disconnect\|Read\|Receive\|Send\|Write\|Block\|Grant\|Protect\|Revoke\|Unblock\|Unprotect\|Use\|ForEach\|Sort\|Tee\|Where)+(-)[\\w\\d]+ | | highlight.js:14:17:14:52 | [a-z0-9&#*=?@\\\\><:,()$[\\]_.{}!+%^-]+ | Strings with many repetitions of '!' can start matching anywhere after the start of the preceeding ([ ]*[a-z0-9&#*=?@\\\\><:,()$[\\]_.{}!+%^-]+)+ | @@ -39,20 +39,20 @@ | lib/snapdragon.js:23:22:23:23 | a* | Strings starting with 'a' and with many repetitions of 'a' can start matching anywhere after the start of the preceeding aa*$ | | lib/subLib4/factory.js:8:3:8:4 | f* | Strings with many repetitions of 'f' can start matching anywhere after the start of the preceeding f*g | | lib/sublib/factory.js:13:14:13:15 | f* | Strings with many repetitions of 'f' can start matching anywhere after the start of the preceeding f*g | -| polynomial-redos.js:7:24:7:26 | \\s+ | Strings with many repetitions of ' ' can start matching anywhere after the start of the preceeding \\s+$ | +| polynomial-redos.js:7:24:7:26 | \\s+ | Strings with many repetitions of '\\t' can start matching anywhere after the start of the preceeding \\s+$ | | polynomial-redos.js:8:17:8:18 | * | Strings with many repetitions of ' ' can start matching anywhere after the start of the preceeding *, * | -| polynomial-redos.js:9:19:9:21 | \\s* | Strings with many repetitions of ' ' can start matching anywhere after the start of the preceeding \\s*\\n\\s* | +| polynomial-redos.js:9:19:9:21 | \\s* | Strings with many repetitions of '\\t' can start matching anywhere after the start of the preceeding \\s*\\n\\s* | | polynomial-redos.js:11:19:11:20 | .* | Strings with many repetitions of 'a' can start matching anywhere after the start of the preceeding .*[/\\\\] | | polynomial-redos.js:12:19:12:20 | .* | Strings with many repetitions of 'a' can start matching anywhere after the start of the preceeding .*\\. | -| polynomial-redos.js:15:28:15:35 | [\\s\\S]*? | Strings starting with '`' and with many repetitions of ' ' can start matching anywhere after the start of the preceeding `+ | -| polynomial-redos.js:15:41:15:43 | \\s* | Strings starting with '`_' and with many repetitions of ' ' can start matching anywhere after the start of the preceeding \\s* | +| polynomial-redos.js:15:28:15:35 | [\\s\\S]*? | Strings starting with '`' and with many repetitions of '\\t' can start matching anywhere after the start of the preceeding `+ | +| polynomial-redos.js:15:41:15:43 | \\s* | Strings starting with '`_' and with many repetitions of '\\t' can start matching anywhere after the start of the preceeding \\s* | | polynomial-redos.js:16:25:16:32 | [\\s\\S]*? | Strings starting with '`' and with many repetitions of '``' can start matching anywhere after the start of the preceeding `+ | | polynomial-redos.js:17:5:17:6 | .* | Strings with many repetitions of ',' can start matching anywhere after the start of the preceeding (.*,)+ | | polynomial-redos.js:17:11:17:12 | .+ | Strings starting with ',' and with many repetitions of ',,' can start matching anywhere after the start of the preceeding .* | | polynomial-redos.js:18:83:18:100 | [\\u0600-\\u06FF\\/]+ | Strings with many repetitions of '/' can start matching anywhere after the start of the preceeding [\\u0600-\\u06FF\\/]+(\\s*?[\\u0600-\\u06FF]+){1,2} | | polynomial-redos.js:19:17:19:22 | [0-9]* | Strings with many repetitions of '0' can start matching anywhere after the start of the preceeding [0-9]*['a-z\\u00A0-\\u05FF\\u0700-\\uD7FF\\uF900-\\uFDCF\\uFDF0-\\uFFEF]{1,256} | -| polynomial-redos.js:20:56:20:58 | \\d+ | Strings with many repetitions of '9' can start matching anywhere after the start of the preceeding \\d* | -| polynomial-redos.js:22:57:22:59 | \\d+ | Strings with many repetitions of '9' can start matching anywhere after the start of the preceeding \\d* | +| polynomial-redos.js:20:56:20:58 | \\d+ | Strings with many repetitions of '0' can start matching anywhere after the start of the preceeding \\d* | +| polynomial-redos.js:22:57:22:59 | \\d+ | Strings with many repetitions of '0' can start matching anywhere after the start of the preceeding \\d* | | polynomial-redos.js:25:37:25:56 | [a-zA-Z0-9+\\/ \\t\\n]+ | Strings starting with '-\\t' and with many repetitions of '\\t\\t' can start matching anywhere after the start of the preceeding [ \\t]+ | | polynomial-redos.js:25:63:25:64 | .* | Strings starting with '-\\t\\t' and with many repetitions of '=' can start matching anywhere after the start of the preceeding [=]* | | polynomial-redos.js:30:19:30:22 | [?]+ | Strings with many repetitions of '?' can start matching anywhere after the start of the preceeding [?]+.*$ | @@ -68,7 +68,7 @@ | polynomial-redos.js:38:34:38:35 | .* | Strings starting with ' | | polynomial-redos.js:75:35:75:36 | .* | Strings starting with '([0-9]\|[ ]\|[-]\|[\\(]\|[\\)]\|ext.\|[,])+)([ ]\|[:]\|\\t\|[-])*(?Home\|Office\|Work\|Away\|Fax\|FAX\|Phone) | | regexplib/address.js:85:51:85:67 | ([ ]\|[:]\|\\t\|[-])* | Strings starting with '0' and with many repetitions of ' ' can start matching anywhere after the start of the preceeding ([0-9]\|[ ]\|[-]\|[\\(]\|[\\)]\|ext.\|[,])+ | -| regexplib/address.js:93:3:93:5 | \\s* | Strings with many repetitions of ' ' can start matching anywhere after the start of the preceeding (\\s*\\(?0\\d{4}\\)?(\\s*\|-)\\d{3}(\\s*\|-)\\d{3}\\s*) | -| regexplib/address.js:93:48:93:50 | \\s* | Strings with many repetitions of ' ' can start matching anywhere after the start of the preceeding (\\s*\\(?0\\d{3}\\)?(\\s*\|-)\\d{3}(\\s*\|-)\\d{4}\\s*) | -| regexplib/address.js:93:93:93:95 | \\s* | Strings with many repetitions of ' ' can start matching anywhere after the start of the preceeding (\\s*(7\|8)(\\d{7}\|\\d{3}(\\-\|\\s{1})\\d{4})\\s*) | +| regexplib/address.js:93:3:93:5 | \\s* | Strings with many repetitions of '\\t' can start matching anywhere after the start of the preceeding (\\s*\\(?0\\d{4}\\)?(\\s*\|-)\\d{3}(\\s*\|-)\\d{3}\\s*) | +| regexplib/address.js:93:48:93:50 | \\s* | Strings with many repetitions of '\\t' can start matching anywhere after the start of the preceeding (\\s*\\(?0\\d{3}\\)?(\\s*\|-)\\d{3}(\\s*\|-)\\d{4}\\s*) | +| regexplib/address.js:93:93:93:95 | \\s* | Strings with many repetitions of '\\t' can start matching anywhere after the start of the preceeding (\\s*(7\|8)(\\d{7}\|\\d{3}(\\-\|\\s{1})\\d{4})\\s*) | | regexplib/address.js:95:379:95:755 | [a-zA-Z0-9ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝßàáâãäåæçèéêëìíîïñòóôõöøùúûüýÿ\\.\\,\\-\\/\\' ]+ | Strings starting with '#' and with many repetitions of '#' can start matching anywhere after the start of the preceeding [a-zA-Z0-9ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝßàáâãäåæçèéêëìíîïñòóôõöøùúûüýÿ\\.\\,\\-\\/\\']+ | | regexplib/address.js:99:19:99:24 | [0-9]* | Strings with many repetitions of '0' can start matching anywhere after the start of the preceeding [0-9]* | | regexplib/email.js:1:16:1:22 | [-.\\w]* | Strings starting with '0' and with many repetitions of '0' can start matching anywhere after the start of the preceeding ([-.\\w]*[0-9a-zA-Z])* | -| regexplib/email.js:2:5:2:12 | [-._\\w]* | Strings starting with 'a' and with many repetitions of '0' can start matching anywhere after the start of the preceeding (\\w[-._\\w]*\\w@\\w[-._\\w]*\\w\\.\\w{2,3}) | +| regexplib/email.js:2:5:2:12 | [-._\\w]* | Strings starting with '0' and with many repetitions of '0' can start matching anywhere after the start of the preceeding (\\w[-._\\w]*\\w@\\w[-._\\w]*\\w\\.\\w{2,3}) | | regexplib/email.js:5:24:5:35 | [a-zA-Z0-9]+ | Strings starting with '0' and with many repetitions of '0' can start matching anywhere after the start of the preceeding ([_\\.\\-]?[a-zA-Z0-9]+)* | | regexplib/email.js:5:63:5:74 | [a-zA-Z0-9]+ | Strings starting with '0@0' and with many repetitions of '0' can start matching anywhere after the start of the preceeding [A-Za-z0-9]+ | | regexplib/email.js:8:16:8:49 | [^ \\t\\(\\)\\<\\>@,;\\:\\\\\\"\\.\\[\\]\\r\\n]+ | Strings with many repetitions of '!' can start matching anywhere after the start of the preceeding (?(?:(?:[^ \\t\\(\\)\\<\\>@,;\\:\\\\\\"\\.\\[\\]\\r\\n]+)\|(?:\\"(?:(?:[^\\"\\\\\\r\\n])\|(?:\\\\.))*\\"))(?:\\.(?:(?:[^ \\t\\(\\)\\<\\>@,;\\:\\\\\\"\\.\\[\\]\\r\\n]+)\|(?:\\"(?:(?:[^\\"\\\\\\r\\n])\|(?:\\\\.))*\\")))*)@(?(?:(?:[^ \\t\\(\\)\\<\\>@,;\\:\\\\\\"\\.\\[\\]\\r\\n]+)\|(?:\\[(?:(?:[^\\[\\]\\\\\\r\\n])\|(?:\\\\.))*\\]))(?:\\.(?:(?:[^ \\t\\(\\)\\<\\>@,;\\:\\\\\\"\\.\\[\\]\\r\\n]+)\|(?:\\[(?:(?:[^\\[\\]\\\\\\r\\n])\|(?:\\\\.))*\\])))*) | @@ -163,20 +163,20 @@ | regexplib/email.js:8:89:8:174 | (?:\\.(?:(?:[^ \\t\\(\\)\\<\\>@,;\\:\\\\\\"\\.\\[\\]\\r\\n]+)\|(?:\\"(?:(?:[^\\"\\\\\\r\\n])\|(?:\\\\.))*\\")))* | Strings starting with '!' and with many repetitions of '.!' can start matching anywhere after the start of the preceeding (?(?:(?:[^ \\t\\(\\)\\<\\>@,;\\:\\\\\\"\\.\\[\\]\\r\\n]+)\|(?:\\"(?:(?:[^\\"\\\\\\r\\n])\|(?:\\\\.))*\\"))(?:\\.(?:(?:[^ \\t\\(\\)\\<\\>@,;\\:\\\\\\"\\.\\[\\]\\r\\n]+)\|(?:\\"(?:(?:[^\\"\\\\\\r\\n])\|(?:\\\\.))*\\")))*)@(?(?:(?:[^ \\t\\(\\)\\<\\>@,;\\:\\\\\\"\\.\\[\\]\\r\\n]+)\|(?:\\[(?:(?:[^\\[\\]\\\\\\r\\n])\|(?:\\\\.))*\\]))(?:\\.(?:(?:[^ \\t\\(\\)\\<\\>@,;\\:\\\\\\"\\.\\[\\]\\r\\n]+)\|(?:\\[(?:(?:[^\\[\\]\\\\\\r\\n])\|(?:\\\\.))*\\])))*) | | regexplib/email.js:8:100:8:133 | [^ \\t\\(\\)\\<\\>@,;\\:\\\\\\"\\.\\[\\]\\r\\n]+ | Strings starting with '!.' and with many repetitions of '!.!' can start matching anywhere after the start of the preceeding (?(?:(?:[^ \\t\\(\\)\\<\\>@,;\\:\\\\\\"\\.\\[\\]\\r\\n]+)\|(?:\\"(?:(?:[^\\"\\\\\\r\\n])\|(?:\\\\.))*\\"))(?:\\.(?:(?:[^ \\t\\(\\)\\<\\>@,;\\:\\\\\\"\\.\\[\\]\\r\\n]+)\|(?:\\"(?:(?:[^\\"\\\\\\r\\n])\|(?:\\\\.))*\\")))*)@(?(?:(?:[^ \\t\\(\\)\\<\\>@,;\\:\\\\\\"\\.\\[\\]\\r\\n]+)\|(?:\\[(?:(?:[^\\[\\]\\\\\\r\\n])\|(?:\\\\.))*\\]))(?:\\.(?:(?:[^ \\t\\(\\)\\<\\>@,;\\:\\\\\\"\\.\\[\\]\\r\\n]+)\|(?:\\[(?:(?:[^\\[\\]\\\\\\r\\n])\|(?:\\\\.))*\\])))*) | | regexplib/email.js:8:141:8:168 | (?:(?:[^\\"\\\\\\r\\n])\|(?:\\\\.))* | Strings starting with '!."' and with many repetitions of '".!."' can start matching anywhere after the start of the preceeding (?(?:(?:[^ \\t\\(\\)\\<\\>@,;\\:\\\\\\"\\.\\[\\]\\r\\n]+)\|(?:\\"(?:(?:[^\\"\\\\\\r\\n])\|(?:\\\\.))*\\"))(?:\\.(?:(?:[^ \\t\\(\\)\\<\\>@,;\\:\\\\\\"\\.\\[\\]\\r\\n]+)\|(?:\\"(?:(?:[^\\"\\\\\\r\\n])\|(?:\\\\.))*\\")))*)@(?(?:(?:[^ \\t\\(\\)\\<\\>@,;\\:\\\\\\"\\.\\[\\]\\r\\n]+)\|(?:\\[(?:(?:[^\\[\\]\\\\\\r\\n])\|(?:\\\\.))*\\]))(?:\\.(?:(?:[^ \\t\\(\\)\\<\\>@,;\\:\\\\\\"\\.\\[\\]\\r\\n]+)\|(?:\\[(?:(?:[^\\[\\]\\\\\\r\\n])\|(?:\\\\.))*\\])))*) | -| regexplib/email.js:12:2:12:4 | \\w+ | Strings with many repetitions of 'a' can start matching anywhere after the start of the preceeding \\w+([-+.]\\w+)*@\\w+([-.]\\w+)*\\.\\w+([-.]\\w+)*([,;]\\s*\\w+([-+.]\\w+)*@\\w+([-.]\\w+)*\\.\\w+([-.]\\w+)*)* | -| regexplib/email.js:12:5:12:15 | ([-+.]\\w+)* | Strings starting with 'a' and with many repetitions of '+a' can start matching anywhere after the start of the preceeding \\w+([-+.]\\w+)*@\\w+([-.]\\w+)*\\.\\w+([-.]\\w+)*([,;]\\s*\\w+([-+.]\\w+)*@\\w+([-.]\\w+)*\\.\\w+([-.]\\w+)*)* | -| regexplib/email.js:12:11:12:13 | \\w+ | Strings starting with 'a+' and with many repetitions of 'a+' can start matching anywhere after the start of the preceeding \\w+([-+.]\\w+)*@\\w+([-.]\\w+)*\\.\\w+([-.]\\w+)*([,;]\\s*\\w+([-+.]\\w+)*@\\w+([-.]\\w+)*\\.\\w+([-.]\\w+)*)* | +| regexplib/email.js:12:2:12:4 | \\w+ | Strings with many repetitions of '0' can start matching anywhere after the start of the preceeding \\w+([-+.]\\w+)*@\\w+([-.]\\w+)*\\.\\w+([-.]\\w+)*([,;]\\s*\\w+([-+.]\\w+)*@\\w+([-.]\\w+)*\\.\\w+([-.]\\w+)*)* | +| regexplib/email.js:12:5:12:15 | ([-+.]\\w+)* | Strings starting with '0' and with many repetitions of '+0' can start matching anywhere after the start of the preceeding \\w+([-+.]\\w+)*@\\w+([-.]\\w+)*\\.\\w+([-.]\\w+)*([,;]\\s*\\w+([-+.]\\w+)*@\\w+([-.]\\w+)*\\.\\w+([-.]\\w+)*)* | +| regexplib/email.js:12:11:12:13 | \\w+ | Strings starting with '0+' and with many repetitions of '0+' can start matching anywhere after the start of the preceeding \\w+([-+.]\\w+)*@\\w+([-.]\\w+)*\\.\\w+([-.]\\w+)*([,;]\\s*\\w+([-+.]\\w+)*@\\w+([-.]\\w+)*\\.\\w+([-.]\\w+)*)* | | regexplib/email.js:13:47:13:49 | \\s+ | Strings starting with 'A' and with many repetitions of ' ' can start matching anywhere after the start of the preceeding [a-zA-Z]* | -| regexplib/email.js:15:6:15:13 | [\\w-\\.]* | Strings starting with 'a' and with many repetitions of '0' can start matching anywhere after the start of the preceeding \\w+ | -| regexplib/email.js:15:28:15:30 | \\w* | Strings starting with 'a@a' and with many repetitions of 'aa' can start matching anywhere after the start of the preceeding \\w+ | -| regexplib/email.js:20:3:20:6 | \\w+? | Strings with many repetitions of 'a' can start matching anywhere after the start of the preceeding (\\w+?@\\w+?\\x2E.+) | +| regexplib/email.js:15:6:15:13 | [\\w-\\.]* | Strings starting with '0' and with many repetitions of '0' can start matching anywhere after the start of the preceeding \\w+ | +| regexplib/email.js:15:28:15:30 | \\w* | Strings starting with '0@0' and with many repetitions of '00' can start matching anywhere after the start of the preceeding \\w+ | +| regexplib/email.js:20:3:20:6 | \\w+? | Strings with many repetitions of '0' can start matching anywhere after the start of the preceeding (\\w+?@\\w+?\\x2E.+) | | regexplib/email.js:25:67:25:78 | [a-zA-Z0-9]+ | Strings starting with '0' and with many repetitions of '0' can start matching anywhere after the start of the preceeding ([_\\.\\-]?[a-zA-Z0-9]+)* | | regexplib/email.js:25:106:25:117 | [a-zA-Z0-9]+ | Strings starting with '0@0' and with many repetitions of '0' can start matching anywhere after the start of the preceeding [A-Za-z0-9]+ | | regexplib/email.js:25:212:25:223 | [a-zA-Z0-9]+ | Strings with many repetitions of '0' can start matching anywhere after the start of the preceeding ([_\\.\\-]?[a-zA-Z0-9]+)* | | regexplib/email.js:25:251:25:262 | [a-zA-Z0-9]+ | Strings with many repetitions of '0' can start matching anywhere after the start of the preceeding [A-Za-z0-9]+ | -| regexplib/email.js:28:2:28:4 | \\w+ | Strings with many repetitions of 'a' can start matching anywhere after the start of the preceeding \\w+[\\w-\\.]*\\@\\w+((-\\w+)\|(\\w*))\\.[a-z]{2,3}$ | -| regexplib/email.js:28:5:28:12 | [\\w-\\.]* | Strings starting with 'a' and with many repetitions of '0' can start matching anywhere after the start of the preceeding \\w+ | -| regexplib/email.js:28:27:28:29 | \\w* | Strings starting with 'a@a' and with many repetitions of 'aa' can start matching anywhere after the start of the preceeding \\w+ | +| regexplib/email.js:28:2:28:4 | \\w+ | Strings with many repetitions of '0' can start matching anywhere after the start of the preceeding \\w+[\\w-\\.]*\\@\\w+((-\\w+)\|(\\w*))\\.[a-z]{2,3}$ | +| regexplib/email.js:28:5:28:12 | [\\w-\\.]* | Strings starting with '0' and with many repetitions of '0' can start matching anywhere after the start of the preceeding \\w+ | +| regexplib/email.js:28:27:28:29 | \\w* | Strings starting with '0@0' and with many repetitions of '00' can start matching anywhere after the start of the preceeding \\w+ | | regexplib/email.js:28:73:28:87 | [0-9a-zA-Z'\\.]+ | Strings with many repetitions of ''' can start matching anywhere after the start of the preceeding ([0-9a-zA-Z'\\.]+)@([0-9a-zA-Z']+)\\.([0-9a-zA-Z']+)$ | | regexplib/email.js:28:125:28:139 | [0-9a-zA-Z'\\.]+ | Strings with many repetitions of ''' can start matching anywhere after the start of the preceeding ([0-9a-zA-Z'\\.]+)@([0-9a-zA-Z']+)\\*+$ | | regexplib/email.js:29:2:29:7 | [\\w-]+ | Strings with many repetitions of '-' can start matching anywhere after the start of the preceeding [\\w-]+@([\\w-]+\\.)+[\\w-]+ | @@ -191,11 +191,11 @@ | regexplib/markup.js:1:21:1:22 | .* | Strings starting with '<=' and with many repetitions of '==' can start matching anywhere after the start of the preceeding .* | | regexplib/markup.js:1:40:1:44 | [^<]* | Strings starting with '<=.jpg' and with many repetitions of ';' can start matching anywhere after the start of the preceeding .* | | regexplib/markup.js:2:3:2:7 | [^>]* | Strings starting with '<' and with many repetitions of '<' can start matching anywhere after the start of the preceeding <[^>]*> | -| regexplib/markup.js:3:440:3:456 | (\\s(?.+?))* | Strings starting with '.+?))* | +| regexplib/markup.js:3:440:3:456 | (\\s(?.+?))* | Strings starting with '.+?))* | | regexplib/markup.js:5:1525:5:1527 | \\s* | Strings starting with '?'DateLiteral' ?# Per the VB Spec : DateLiteral ::= '#' DateOrTime '#' # ?'DateOrTime' DateValue ?# TimeValue ::= HourValue : MinuteValue 10 ?# Hour 01 - 24 : 60 ?# Minute 01 - 60 : ?# Optional Minute :01 - :60 ' and with many repetitions of ' ' can start matching anywhere after the start of the preceeding \\s* | -| regexplib/markup.js:6:11:6:25 | [\\w\\*\\)\\(\\,\\s]+ | Strings starting with 'SELECT ' and with many repetitions of 'SELECT\\t' can start matching anywhere after the start of the preceeding (SELECT\\s[\\w\\*\\)\\(\\,\\s]+\\sFROM\\s[\\w]+) | -| regexplib/markup.js:6:99:6:113 | [\\s\\w\\d\\)\\(\\,]* | Strings starting with ' INSERT INTO 0' and with many repetitions of '0' can start matching anywhere after the start of the preceeding [\\d\\w]+ | +| regexplib/markup.js:6:11:6:25 | [\\w\\*\\)\\(\\,\\s]+ | Strings starting with 'SELECT\\t' and with many repetitions of 'SELECT\\t' can start matching anywhere after the start of the preceeding (SELECT\\s[\\w\\*\\)\\(\\,\\s]+\\sFROM\\s[\\w]+) | +| regexplib/markup.js:6:99:6:113 | [\\s\\w\\d\\)\\(\\,]* | Strings starting with ' INSERT\\tINTO\\t0' and with many repetitions of '0' can start matching anywhere after the start of the preceeding [\\d\\w]+ | | regexplib/markup.js:7:15:7:21 | [^\\\\"]* | Strings starting with '"!' and with many repetitions of '\\\\"!!' can start matching anywhere after the start of the preceeding "([^"](?:\\\\.\|[^\\\\"]*)*)" | | regexplib/markup.js:9:6:9:13 | [\\s\\S]*? | Strings starting with ' | @@ -209,44 +209,44 @@ | regexplib/markup.js:14:24:14:25 | .* | Strings starting with '<>' and with many repetitions of '>a' can start matching anywhere after the start of the preceeding .* | | regexplib/markup.js:15:16:15:18 | .*? | Strings starting with ' | | regexplib/markup.js:16:5:16:9 | [^>]* | Strings starting with 'src' and with many repetitions of 'src' can start matching anywhere after the start of the preceeding src[^>]*[^/].(?:jpg\|bmp\|gif)(?:\\"\|\\') | -| regexplib/markup.js:17:8:17:24 | (\\s(\\w*=".*?")?)* | Strings starting with '' and with many repetitions of '">' can start matching anywhere after the start of the preceeding .*? | -| regexplib/markup.js:19:2:19:12 | ([^'("\|')]*)("\|')){1}\|content\\s*=\\s*("\|')(?[^'("\|')]*)("\|')\|scheme\\s*=\\s*("\|')(?[^'("\|')]*)("\|')) | -| regexplib/markup.js:19:8:19:10 | \\s+ | Strings starting with '[^'("\|')]*)("\|')){1}\|content\\s*=\\s*("\|')(?[^'("\|')]*)("\|')\|scheme\\s*=\\s*("\|')(?[^'("\|')]*)("\|')) | +| regexplib/markup.js:17:8:17:24 | (\\s(\\w*=".*?")?)* | Strings starting with '<0' and with many repetitions of '\\t=""' can start matching anywhere after the start of the preceeding .*? | +| regexplib/markup.js:17:12:17:14 | \\w* | Strings starting with '<0\\t' and with many repetitions of '=""\\t0' can start matching anywhere after the start of the preceeding .*? | +| regexplib/markup.js:17:17:17:19 | .*? | Strings starting with '<0\\t="' and with many repetitions of '\\t=""' can start matching anywhere after the start of the preceeding \\w* | +| regexplib/markup.js:17:40:17:42 | .*? | Strings starting with '<0>' and with many repetitions of '">' can start matching anywhere after the start of the preceeding .*? | +| regexplib/markup.js:19:2:19:12 | ([^'("\|')]*)("\|')){1}\|content\\s*=\\s*("\|')(?[^'("\|')]*)("\|')\|scheme\\s*=\\s*("\|')(?[^'("\|')]*)("\|')) | +| regexplib/markup.js:19:8:19:10 | \\s+ | Strings starting with '[^'("\|')]*)("\|')){1}\|content\\s*=\\s*("\|')(?[^'("\|')]*)("\|')\|scheme\\s*=\\s*("\|')(?[^'("\|')]*)("\|')) | | regexplib/markup.js:20:52:20:53 | .* | Strings with many repetitions of '=color' can start matching anywhere after the start of the preceeding [^>]+ | | regexplib/markup.js:20:155:20:156 | '+ | Strings with many repetitions of '''' can start matching anywhere after the start of the preceeding '+ | | regexplib/markup.js:20:197:20:198 | "+ | Strings with many repetitions of '""' can start matching anywhere after the start of the preceeding "+ | | regexplib/markup.js:20:245:20:247 | .*? | Strings with many repetitions of 'color: # IF found THEN move ahead "" # single or double # or no quotes\\t' can start matching anywhere after the start of the preceeding .*? | | regexplib/markup.js:20:274:20:276 | .*? | Strings starting with ']+color.*>) #IF\\/THEN lookahead color in tag (.*?color\\s*?[=\|:]\\s*?) # IF found THEN move ahead ('+\\#*?[\\w\\s]*'+ # CAPTURE ColorName\\/Hex \|"+\\#*?[\\w\\s]*"+ # single or double \|\\#*\\w*\\b) # or no quotes\t.*?> # & move to end of tag \|.*?> # ELSE move to end of Tag ) # Close the If\\/Then lookahead # Use Multiline and IgnoreCase # Replace the matches from RE with MatchEvaluator below: # if m.Groups(1).Value<>"" then # Return "" # else # Return "" # end if | -| regexplib/markup.js:24:39:24:41 | \\s+ | Strings starting with '<A' and with many repetitions of ' - != ' can start matching anywhere after the start of the preceeding \\s* | -| regexplib/markup.js:24:43:24:45 | \\S+ | Strings starting with '<A ' and with many repetitions of '- !=' can start matching anywhere after the start of the preceeding \\s* | -| regexplib/markup.js:24:48:24:50 | \\s* | Strings starting with '<A !' and with many repetitions of ' =- ! ' can start matching anywhere after the start of the preceeding \\s+ | -| regexplib/markup.js:24:52:24:54 | \\s* | Strings starting with '<A !=' and with many repetitions of '- !=' can start matching anywhere after the start of the preceeding \\S+ | +| regexplib/markup.js:24:39:24:41 | \\s+ | Strings starting with '<A' and with many repetitions of '\\t-\\t!=\\t' can start matching anywhere after the start of the preceeding \\s* | +| regexplib/markup.js:24:43:24:45 | \\S+ | Strings starting with '<A\\t' and with many repetitions of '-\\t!=' can start matching anywhere after the start of the preceeding \\s* | +| regexplib/markup.js:24:48:24:50 | \\s* | Strings starting with '<A\\t!' and with many repetitions of '\\t=-\\t!\\t' can start matching anywhere after the start of the preceeding \\s+ | +| regexplib/markup.js:24:52:24:54 | \\s* | Strings starting with '<A\\t!=' and with many repetitions of '-\\t!=' can start matching anywhere after the start of the preceeding \\S+ | | regexplib/markup.js:25:11:25:15 | [^>]* | Strings starting with ']*\\son\\w+=(\\w+\|'[^']*'\|"[^"]*")[^>]*> | -| regexplib/markup.js:25:45:25:49 | [^>]* | Strings starting with ']* | Strings starting with ']* | Strings starting with '<' and with many repetitions of '<' can start matching anywhere after the start of the preceeding <[^>]*name[\\s]*=[\\s]*"?[^\\w_]*"?[^>]*> | | regexplib/markup.js:27:34:27:38 | [^>]* | Strings starting with ']* | Strings starting with '<' and with many repetitions of ']*)(\\s[^<]*)> | -| regexplib/markup.js:37:15:37:19 | [\\w]* | Strings starting with '[a' and with many repetitions of '0' can start matching anywhere after the start of the preceeding \\w+ | -| regexplib/markup.js:40:23:40:25 | \\w+ | Strings with many repetitions of 'a' can start matching anywhere after the start of the preceeding (\\/?(?\\w+))+ | -| regexplib/markup.js:40:59:40:61 | \\s* | Strings starting with 'a[' and with many repetitions of ' ' can start matching anywhere after the start of the preceeding \\s* | +| regexplib/markup.js:37:15:37:19 | [\\w]* | Strings starting with '[0' and with many repetitions of '0' can start matching anywhere after the start of the preceeding \\w+ | +| regexplib/markup.js:40:23:40:25 | \\w+ | Strings with many repetitions of '0' can start matching anywhere after the start of the preceeding (\\/?(?\\w+))+ | +| regexplib/markup.js:40:59:40:61 | \\s* | Strings starting with '0[' and with many repetitions of '\\t' can start matching anywhere after the start of the preceeding \\s* | | regexplib/markup.js:43:11:43:15 | [^>]* | Strings starting with ']*\\son\\w+=(\\w+\|'[^']*'\|"[^"]*")[^>]*> | -| regexplib/markup.js:43:45:43:49 | [^>]* | Strings starting with ']* | Strings starting with ']* | Strings starting with '<' and with many repetitions of '<' can start matching anywhere after the start of the preceeding <[^>]*name[\\s]*=[\\s]*"?[^\\w_]*"?[^>]*> | | regexplib/markup.js:44:34:44:38 | [^>]* | Strings starting with ' | -| regexplib/markup.js:53:15:53:19 | [\\w]* | Strings starting with '[a' and with many repetitions of '0' can start matching anywhere after the start of the preceeding \\w+ | -| regexplib/markup.js:56:23:56:25 | \\w+ | Strings with many repetitions of 'a' can start matching anywhere after the start of the preceeding (\\/?(?\\w+))+ | -| regexplib/markup.js:56:59:56:61 | \\s* | Strings starting with 'a[' and with many repetitions of ' ' can start matching anywhere after the start of the preceeding \\s* | +| regexplib/markup.js:53:15:53:19 | [\\w]* | Strings starting with '[0' and with many repetitions of '0' can start matching anywhere after the start of the preceeding \\w+ | +| regexplib/markup.js:56:23:56:25 | \\w+ | Strings with many repetitions of '0' can start matching anywhere after the start of the preceeding (\\/?(?\\w+))+ | +| regexplib/markup.js:56:59:56:61 | \\s* | Strings starting with '0[' and with many repetitions of '\\t' can start matching anywhere after the start of the preceeding \\s* | | regexplib/markup.js:61:74:61:79 | [0-9]+ | Strings starting with '.+)(\\.)(?[^\\.]*)(\\()(?[^\\)]*)(\\))((\\sin\\s)(?.+)(:line )(?[\\d]*))? | -| regexplib/misc.js:95:71:95:76 | [^\\)]* | Strings starting with 'at a.(' and with many repetitions of '((' can start matching anywhere after the start of the preceeding .+ | -| regexplib/misc.js:95:103:95:104 | .+ | Strings starting with 'at a.() in ' and with many repetitions of '() in -' can start matching anywhere after the start of the preceeding .+ | +| regexplib/misc.js:95:25:95:26 | .+ | Strings starting with 'at\\t' and with many repetitions of 'at\\ta' can start matching anywhere after the start of the preceeding (at\\s)(?.+)(\\.)(?[^\\.]*)(\\()(?[^\\)]*)(\\))((\\sin\\s)(?.+)(:line )(?[\\d]*))? | +| regexplib/misc.js:95:71:95:76 | [^\\)]* | Strings starting with 'at\\ta.(' and with many repetitions of '((' can start matching anywhere after the start of the preceeding .+ | +| regexplib/misc.js:95:103:95:104 | .+ | Strings starting with 'at\\ta.()\\tin\\t' and with many repetitions of '()\\tin\\t-' can start matching anywhere after the start of the preceeding .+ | | regexplib/misc.js:101:52:101:70 | [a-z0-9\\/\\.\\?\\=\\&]* | Strings starting with '".htm' and with many repetitions of '.asp' can start matching anywhere after the start of the preceeding [a-z0-9\\/\\.\\?\\=\\&]* | -| regexplib/misc.js:112:3:112:5 | \\s* | Strings with many repetitions of ' ' can start matching anywhere after the start of the preceeding (\\s*\\(?0\\d{4}\\)?\\s*\\d{6}\\s*) | -| regexplib/misc.js:112:32:112:34 | \\s* | Strings with many repetitions of ' ' can start matching anywhere after the start of the preceeding (\\s*\\(?0\\d{3}\\)?\\s*\\d{3}\\s*\\d{4}\\s*) | +| regexplib/misc.js:112:3:112:5 | \\s* | Strings with many repetitions of '\\t' can start matching anywhere after the start of the preceeding (\\s*\\(?0\\d{4}\\)?\\s*\\d{6}\\s*) | +| regexplib/misc.js:112:32:112:34 | \\s* | Strings with many repetitions of '\\t' can start matching anywhere after the start of the preceeding (\\s*\\(?0\\d{3}\\)?\\s*\\d{3}\\s*\\d{4}\\s*) | | regexplib/misc.js:114:6:114:8 | \\\|+ | Strings starting with 'a' and with many repetitions of '\|' can start matching anywhere after the start of the preceeding .+ | | regexplib/misc.js:116:3:116:4 | .* | Strings starting with '{' and with many repetitions of '{' can start matching anywhere after the start of the preceeding {.*} | | regexplib/misc.js:117:25:117:26 | .+ | Strings starting with '{a}' and with many repetitions of 'a)' can start matching anywhere after the start of the preceeding .+ | -| regexplib/misc.js:119:20:119:22 | \\w+ | Strings with many repetitions of 'a' can start matching anywhere after the start of the preceeding (NOT)?(\\s*\\(*)\\s*(\\w+)\\s*(=\|<>\|<\|>\|LIKE\|IN)\\s*(\\(([^\\)]*)\\)\|'([^']*)'\|(-?\\d*\\.?\\d+))(\\s*\\)*\\s*)(AND\|OR)? | +| regexplib/misc.js:119:20:119:22 | \\w+ | Strings with many repetitions of '0' can start matching anywhere after the start of the preceeding (NOT)?(\\s*\\(*)\\s*(\\w+)\\s*(=\|<>\|<\|>\|LIKE\|IN)\\s*(\\(([^\\)]*)\\)\|'([^']*)'\|(-?\\d*\\.?\\d+))(\\s*\\)*\\s*)(AND\|OR)? | +| regexplib/misc.js:119:52:119:57 | [^\\)]* | Strings starting with '0=(' and with many repetitions of '0<((' can start matching anywhere after the start of the preceeding (NOT)?(\\s*\\(*)\\s*(\\w+)\\s*(=\|<>\|<\|>\|LIKE\|IN)\\s*(\\(([^\\)]*)\\)\|'([^']*)'\|(-?\\d*\\.?\\d+))(\\s*\\)*\\s*)(AND\|OR)? | | regexplib/misc.js:123:36:123:38 | .*? | Strings starting with '?se[A' and with many repetitions of '?se[Aa' can start matching anywhere after the start of the preceeding (?s)(?:\\e\\[(?:(\\d+);?)*([A-Za-z])(.*?))(?=\\e\\[\|\\z) | | regexplib/misc.js:126:15:126:20 | [a-z]+ | Strings starting with 'a' and with many repetitions of 'aa' can start matching anywhere after the start of the preceeding [a-z]+ | | regexplib/misc.js:141:15:141:19 | [^;]+ | Strings starting with '{\\\\f\\\\' and with many repetitions of '{\\\\f\\\\:' can start matching anywhere after the start of the preceeding (\\{\\\\f\\d*)\\\\([^;]+;) | | regexplib/misc.js:144:52:144:70 | [a-z0-9\\/\\.\\?\\=\\&]* | Strings starting with '".htm' and with many repetitions of '.asp' can start matching anywhere after the start of the preceeding [a-z0-9\\/\\.\\?\\=\\&]* | | regexplib/misc.js:148:12:148:18 | [^\\s>]+ | Strings starting with '<' and with many repetitions of ']+(\\s+[^"'=]+(=("[^"]*")\|('[^\\']*')\|([^\\s"'>]*))?)*\\s*\\/?> | -| regexplib/misc.js:148:20:148:22 | \\s+ | Strings starting with ']+(\\s+[^"'=]+(=("[^"]*")\|('[^\\']*')\|([^\\s"'>]*))?)*\\s*\\/?> | -| regexplib/misc.js:148:44:148:49 | [^\\']* | Strings starting with ']+(\\s+[^"'=]+(=("[^"]*")\|('[^\\']*')\|([^\\s"'>]*))?)*\\s*\\/?> | -| regexplib/misc.js:148:68:148:70 | \\s* | Strings starting with ']+(\\s+[^"'=]+(=("[^"]*")\|('[^\\']*')\|([^\\s"'>]*))?)*\\s*\\/?> | +| regexplib/misc.js:148:44:148:49 | [^\\']* | Strings starting with ']+(\\s+[^"'=]+(=("[^"]*")\|('[^\\']*')\|([^\\s"'>]*))?)*\\s*\\/?> | +| regexplib/misc.js:148:68:148:70 | \\s* | Strings starting with '\|<\|>\|LIKE\|IN)\\s*(\\(([^\\)]*)\\)\|'([^']*)'\|(-?\\d*\\.?\\d+))(\\s*\\)*\\s*)(AND\|OR)? | +| regexplib/strings.js:54:20:54:22 | \\w+ | Strings with many repetitions of '0' can start matching anywhere after the start of the preceeding (NOT)?(\\s*\\(*)\\s*(\\w+)\\s*(=\|<>\|<\|>\|LIKE\|IN)\\s*(\\(([^\\)]*)\\)\|'([^']*)'\|(-?\\d*\\.?\\d+))(\\s*\\)*\\s*)(AND\|OR)? | +| regexplib/strings.js:54:52:54:57 | [^\\)]* | Strings starting with '0=(' and with many repetitions of '0<((' can start matching anywhere after the start of the preceeding (NOT)?(\\s*\\(*)\\s*(\\w+)\\s*(=\|<>\|<\|>\|LIKE\|IN)\\s*(\\(([^\\)]*)\\)\|'([^']*)'\|(-?\\d*\\.?\\d+))(\\s*\\)*\\s*)(AND\|OR)? | | regexplib/strings.js:56:52:56:53 | .+ | Strings starting with 'PRN.' and with many repetitions of '.' can start matching anywhere after the start of the preceeding .* | | regexplib/strings.js:57:36:57:38 | .*? | Strings starting with '?se[A' and with many repetitions of '?se[Aa' can start matching anywhere after the start of the preceeding (?s)(?:\\e\\[(?:(\\d+);?)*([A-Za-z])(.*?))(?=\\e\\[\|\\z) | -| regexplib/strings.js:64:3:64:5 | \\w+ | Strings with many repetitions of 'a' can start matching anywhere after the start of the preceeding (\\w+)\\s+\\1 | +| regexplib/strings.js:64:3:64:5 | \\w+ | Strings with many repetitions of '0' can start matching anywhere after the start of the preceeding (\\w+)\\s+\\1 | | regexplib/strings.js:70:6:70:17 | [a-zA-Z,\\s]+ | Strings with many repetitions of '\\t' can start matching anywhere after the start of the preceeding \\s* | | regexplib/strings.js:70:18:70:20 | \\s* | Strings starting with '\\t' and with many repetitions of '\\t' can start matching anywhere after the start of the preceeding \\s* | -| regexplib/strings.js:72:35:72:37 | \\w* | Strings starting with 'A' and with many repetitions of 'A' can start matching anywhere after the start of the preceeding \\w* | +| regexplib/strings.js:72:35:72:37 | \\w* | Strings starting with 'A' and with many repetitions of '0' can start matching anywhere after the start of the preceeding \\w* | | regexplib/strings.js:72:61:72:63 | \\w+ | Strings starting with 'AA' and with many repetitions of 'A' can start matching anywhere after the start of the preceeding \\w* | -| regexplib/strings.js:72:119:72:121 | \\w* | Strings with many repetitions of 'aA' can start matching anywhere after the start of the preceeding \\w+ | +| regexplib/strings.js:72:119:72:121 | \\w* | Strings with many repetitions of '00' can start matching anywhere after the start of the preceeding \\w+ | | regexplib/strings.js:73:2:73:3 | .* | Strings with many repetitions of 'a' can start matching anywhere after the start of the preceeding .*\\$AVE | | regexplib/strings.js:74:2:74:3 | .* | Strings with many repetitions of 'a' can start matching anywhere after the start of the preceeding .*[Pp]re[Ss\\$]cr[iI1]pt.* | | regexplib/strings.js:75:2:75:3 | .* | Strings with many repetitions of 'a' can start matching anywhere after the start of the preceeding .*[Vv][Ii1]agr.* | | regexplib/strings.js:76:2:76:3 | .* | Strings with many repetitions of 'a' can start matching anywhere after the start of the preceeding .*[Oo0][Ee][Mm].* | | regexplib/strings.js:81:36:81:38 | .*? | Strings starting with '?se[A' and with many repetitions of '?se[Aa' can start matching anywhere after the start of the preceeding (?s)(?:\\e\\[(?:(\\d+);?)*([A-Za-z])(.*?))(?=\\e\\[\|\\z) | -| regexplib/strings.js:82:20:82:22 | \\w+ | Strings with many repetitions of 'a' can start matching anywhere after the start of the preceeding (NOT)?(\\s*\\(*)\\s*(\\w+)\\s*(=\|<>\|<\|>\|LIKE\|IN)\\s*(\\(([^\\)]*)\\)\|'([^']*)'\|(-?\\d*\\.?\\d+))(\\s*\\)*\\s*)(AND\|OR)? | +| regexplib/strings.js:82:20:82:22 | \\w+ | Strings with many repetitions of '0' can start matching anywhere after the start of the preceeding (NOT)?(\\s*\\(*)\\s*(\\w+)\\s*(=\|<>\|<\|>\|LIKE\|IN)\\s*(\\(([^\\)]*)\\)\|'([^']*)'\|(-?\\d*\\.?\\d+))(\\s*\\)*\\s*)(AND\|OR)? | +| regexplib/strings.js:82:52:82:57 | [^\\)]* | Strings starting with '0=(' and with many repetitions of '0<((' can start matching anywhere after the start of the preceeding (NOT)?(\\s*\\(*)\\s*(\\w+)\\s*(=\|<>\|<\|>\|LIKE\|IN)\\s*(\\(([^\\)]*)\\)\|'([^']*)'\|(-?\\d*\\.?\\d+))(\\s*\\)*\\s*)(AND\|OR)? | | regexplib/strings.js:88:3:88:12 | [^\\.\\?\\!]* | Strings with many repetitions of ' ' can start matching anywhere after the start of the preceeding ([^\\.\\?\\!]*)[\\.\\?\\!] | | regexplib/strings.js:89:3:89:5 | \\S+ | Strings with many repetitions of '!' can start matching anywhere after the start of the preceeding (\\S+)\\x20{2,}(?=\\S+) | | regexplib/uri.js:3:149:3:151 | .*? | Strings starting with 'ftp:// ' and with many repetitions of '/ /' can start matching anywhere after the start of the preceeding .* | | regexplib/uri.js:3:188:3:190 | \\w+ | Strings starting with 'ftp:// ' and with many repetitions of '00' can start matching anywhere after the start of the preceeding .* | -| regexplib/uri.js:3:193:3:198 | [^\\#]+ | Strings starting with 'ftp:// a=' and with many repetitions of '00=' can start matching anywhere after the start of the preceeding .* | -| regexplib/uri.js:3:206:3:208 | \\w+ | Strings starting with 'ftp:// a="' and with many repetitions of '00' can start matching anywhere after the start of the preceeding .* | -| regexplib/uri.js:3:211:3:213 | \\w+ | Strings starting with 'ftp:// a="a=' and with many repetitions of '00=' can start matching anywhere after the start of the preceeding .* | -| regexplib/uri.js:5:42:5:43 | .* | Strings starting with 'A:\\\\a' and with many repetitions of ' ' can start matching anywhere after the start of the preceeding [\\w ]* | +| regexplib/uri.js:3:193:3:198 | [^\\#]+ | Strings starting with 'ftp:// 0=' and with many repetitions of '00=' can start matching anywhere after the start of the preceeding .* | +| regexplib/uri.js:3:206:3:208 | \\w+ | Strings starting with 'ftp:// 0="' and with many repetitions of '00' can start matching anywhere after the start of the preceeding .* | +| regexplib/uri.js:3:211:3:213 | \\w+ | Strings starting with 'ftp:// 0="0=' and with many repetitions of '00=' can start matching anywhere after the start of the preceeding .* | +| regexplib/uri.js:5:42:5:43 | .* | Strings starting with 'A:\\\\0' and with many repetitions of ' ' can start matching anywhere after the start of the preceeding [\\w ]* | | regexplib/uri.js:14:175:14:213 | [\\d\\w\\.\\/\\%\\+\\-\\=\\&\\?\\:\\\\\\"\\'\\,\\\|\\~\\;]* | Strings with many repetitions of '.ac+' can start matching anywhere after the start of the preceeding [\\d\\w\\.\\/\\+\\-\\?\\:]* | -| regexplib/uri.js:17:42:17:43 | .* | Strings starting with 'A:\\\\a' and with many repetitions of ' ' can start matching anywhere after the start of the preceeding [\\w ]* | +| regexplib/uri.js:17:42:17:43 | .* | Strings starting with 'A:\\\\0' and with many repetitions of ' ' can start matching anywhere after the start of the preceeding [\\w ]* | | regexplib/uri.js:18:4:18:46 | ([A-Za-z0-9'~`!@#$%&^_+=\\(\\){},\\-\\[\\]\\;])+? | Strings starting with 'A' and with many repetitions of 'A' can start matching anywhere after the start of the preceeding \\A([A-Za-z0-9'~`!@#$%&^_+=\\(\\){},\\-\\[\\]\\;])+?([ A-Za-z0-9'~` !@#$%&^_+=\\(\\){},\\-\\[\\];]\|([.]))*?(?(3)(([ A-Za-z0-9'~`!@#$ %&^_+=\\(\\){},\\-\\[\\]\\;]*?)([A-Za-z0-9'~`!@#$%&^_+=\\(\\){},\\-\\[ \\];])+\\z)\|(\\z)) | | regexplib/uri.js:18:47:18:96 | ([ A-Za-z0-9'~` !@#$%&^_+=\\(\\){},\\-\\[\\];]\|([.]))*? | Strings starting with 'A!' and with many repetitions of '!' can start matching anywhere after the start of the preceeding ([A-Za-z0-9'~`!@#$%&^_+=\\(\\){},\\-\\[\\]\\;])+? | | regexplib/uri.js:18:148:18:189 | ([A-Za-z0-9'~`!@#$%&^_+=\\(\\){},\\-\\[ \\];])+ | Strings starting with 'A!?3' and with many repetitions of ' ' can start matching anywhere after the start of the preceeding [ A-Za-z0-9'~`!@#$ %&^_+=\\(\\){},\\-\\[\\]\\;]*? | @@ -387,7 +390,7 @@ | regexplib/uri.js:38:20:38:28 | [a-z0-9]+ | Strings starting with 'a' and with many repetitions of '0' can start matching anywhere after the start of the preceeding [a-z]+ | | regexplib/uri.js:38:35:38:40 | [a-z]+ | Strings starting with 'a.' and with many repetitions of 'aa' can start matching anywhere after the start of the preceeding [a-z0-9]+ | | regexplib/uri.js:38:52:38:60 | [a-z0-9]+ | Strings starting with 'a.a' and with many repetitions of '0' can start matching anywhere after the start of the preceeding [a-z]+ | -| regexplib/uri.js:39:7:39:9 | .*? | Strings starting with 'a' can start matching anywhere after the start of the preceeding .*? | | regexplib/uri.js:41:16:41:31 | [a-zA-Z0-9\\-\\.]+ | Strings starting with '0' and with many repetitions of '00' can start matching anywhere after the start of the preceeding [a-zA-Z0-9]+ | | regexplib/uri.js:44:2:44:4 | .*? | Strings with many repetitions of 'a' can start matching anywhere after the start of the preceeding .*?$(?a]+ | Strings with many repetitions of '=' can start matching anywhere after the start of the preceeding ([^>a]+)* | | tst.js:188:17:188:19 | \\s* | Strings starting with '\\n' and with many repetitions of '\\n' can start matching anywhere after the start of the preceeding (\\n\\s*)+$ | -| tst.js:191:18:191:20 | \\s+ | Strings with many repetitions of ' ' can start matching anywhere after the start of the preceeding .* | +| tst.js:191:18:191:20 | \\s+ | Strings with many repetitions of '\\t' can start matching anywhere after the start of the preceeding .* | | tst.js:191:31:191:35 | [^)]* | Strings starting with '(?#' and with many repetitions of '(?#' can start matching anywhere after the start of the preceeding .* | | tst.js:194:53:194:61 | [a-zA-Z]+ | Strings starting with '{[A(A)' and with many repetitions of 'AA' can start matching anywhere after the start of the preceeding [ a-zA-Z{}]+ | | tst.js:194:68:194:79 | [ a-zA-Z{}]+ | Strings starting with '{[A(A)A:' and with many repetitions of ' A: ' can start matching anywhere after the start of the preceeding \\s* | @@ -475,23 +478,23 @@ | tst.js:227:15:227:22 | ([^X]b)+ | Strings with many repetitions of 'Wb' can start matching anywhere after the start of the preceeding (([^X]b)+)* | | tst.js:239:15:239:19 | (ab)+ | Strings with many repetitions of 'ab' can start matching anywhere after the start of the preceeding ((ab)+)* | | tst.js:248:18:248:19 | A* | Strings with many repetitions of 'A' can start matching anywhere after the start of the preceeding A* | -| tst.js:254:15:254:17 | \\w* | Strings with many repetitions of 'a' can start matching anywhere after the start of the preceeding \\d* | +| tst.js:254:15:254:17 | \\w* | Strings with many repetitions of '0' can start matching anywhere after the start of the preceeding \\d* | | tst.js:254:27:254:29 | \\w* | Strings starting with 'foobarbaz' and with many repetitions of 'foobarbaz' can start matching anywhere after the start of the preceeding \\d* | | tst.js:254:39:254:41 | \\w* | Strings starting with 'foobarbazfoobarbaz' and with many repetitions of 'foobarbaz' can start matching anywhere after the start of the preceeding \\d* | | tst.js:254:51:254:53 | \\w* | Strings starting with 'foobarbazfoobarbazfoobarbaz' and with many repetitions of 'foobarbaz' can start matching anywhere after the start of the preceeding \\d* | | tst.js:254:63:254:65 | \\s* | Strings starting with 'foobarbazfoobarbazfoobarbazfoobarbaz' and with many repetitions of 'foobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbaz' can start matching anywhere after the start of the preceeding \\d* | | tst.js:254:75:254:77 | \\d* | Strings starting with 'foobarbazfoobarbazfoobarbazfoobarbazfoobarbaz' and with many repetitions of 'foobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbaz' can start matching anywhere after the start of the preceeding \\s* | -| tst.js:257:14:257:116 | (.thisisagoddamnlongstringforstresstestingthequery\|\\sthisisagoddamnlongstringforstresstestingthequery)* | Strings with many repetitions of ' thisisagoddamnlongstringforstresstestingthequery' can start matching anywhere after the start of the preceeding (.thisisagoddamnlongstringforstresstestingthequery\|\\sthisisagoddamnlongstringforstresstestingthequery)*- | -| tst.js:260:14:260:77 | (thisisagoddamnlongstringforstresstestingthequery\|this\\w+query)* | Strings with many repetitions of 'thisaquery' can start matching anywhere after the start of the preceeding \\w+ | +| tst.js:257:14:257:116 | (.thisisagoddamnlongstringforstresstestingthequery\|\\sthisisagoddamnlongstringforstresstestingthequery)* | Strings with many repetitions of '\\tthisisagoddamnlongstringforstresstestingthequery' can start matching anywhere after the start of the preceeding (.thisisagoddamnlongstringforstresstestingthequery\|\\sthisisagoddamnlongstringforstresstestingthequery)*- | +| tst.js:260:14:260:77 | (thisisagoddamnlongstringforstresstestingthequery\|this\\w+query)* | Strings with many repetitions of 'this0query' can start matching anywhere after the start of the preceeding \\w+ | | tst.js:260:68:260:70 | \\w+ | Strings starting with 'this' and with many repetitions of 'this' can start matching anywhere after the start of the preceeding (thisisagoddamnlongstringforstresstestingthequery\|this\\w+query)* | | tst.js:263:15:263:117 | (thisisagoddamnlongstringforstresstestingthequery\|imanotherbutunrelatedstringcomparedtotheotherstring)* | Strings with many repetitions of 'thisisagoddamnlongstringforstresstestingthequery' can start matching anywhere after the start of the preceeding (thisisagoddamnlongstringforstresstestingthequery\|imanotherbutunrelatedstringcomparedtotheotherstring)*- | | tst.js:272:21:272:22 | b+ | Strings with many repetitions of 'b' can start matching anywhere after the start of the preceeding (b+)+ | -| tst.js:275:25:275:27 | \\s+ | Strings starting with '/) | polynomial-redos.js:5:16:5:32 | req.query.tainted | polynomial-redos.js:38:2:38:8 | tainted | This $@ that depends on $@ may run slow on strings starting with '/) | polynomial-redos.js:5:16:5:32 | req.query.tainted | polynomial-redos.js:75:2:75:8 | tainted | This $@ that depends on $@ may run slow on strings starting with '<' and with many repetitions of '<'. | polynomial-redos.js:75:18:75:19 | .* | regular expression | polynomial-redos.js:5:16:5:32 | req.query.tainted | a user-provided value | | polynomial-redos.js:75:2:75:39 | tainted ... )".*>/) | polynomial-redos.js:5:16:5:32 | req.query.tainted | polynomial-redos.js:75:2:75:8 | tainted | This $@ that depends on $@ may run slow on strings starting with '\\. ])\|([^\\\\/:\\*\\?"\\\|<>]*[^\\\\/:\\*\\?"\\\|<>\\. ]))? | This part of the regular expression may cause exponential backtracking on strings starting with '!' and containing many repetitions of '!\\\\!'. | | regexplib/misc.js:24:56:24:118 | (([^\\\\/:\\*\\?"\\\|<>\\. ])\|([^\\\\/:\\*\\?"\\\|<>]*[^\\\\/:\\*\\?"\\\|<>\\. ]))? | This part of the regular expression may cause exponential backtracking on strings starting with '!' and containing many repetitions of '!\\\\!'. | | regexplib/misc.js:79:3:79:25 | (\\/w\|\\/W\|[^<>+?$%{}&])+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '/W'. | | regexplib/misc.js:90:4:90:11 | ([a-z])+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'aa'. | -| regexplib/misc.js:123:17:123:19 | \\d+ | This part of the regular expression may cause exponential backtracking on strings starting with '?se[' and containing many repetitions of '9'. | +| regexplib/misc.js:123:17:123:19 | \\d+ | This part of the regular expression may cause exponential backtracking on strings starting with '?se[' and containing many repetitions of '0'. | | regexplib/misc.js:142:3:142:25 | (\\/w\|\\/W\|[^<>+?$%{}&])+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '/W'. | -| regexplib/misc.js:148:20:148:22 | \\s+ | This part of the regular expression may cause exponential backtracking on strings starting with 'a]+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '='. | | tst.js:188:17:188:19 | \\s* | This part of the regular expression may cause exponential backtracking on strings starting with '\\n' and containing many repetitions of '\\n'. | -| tst.js:191:18:191:20 | \\s+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of ' '. | +| tst.js:191:18:191:20 | \\s+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '\\t'. | | tst.js:194:68:194:79 | [ a-zA-Z{}]+ | This part of the regular expression may cause exponential backtracking on strings starting with '{[A(A)A:' and containing many repetitions of ' A:'. | | tst.js:194:81:194:82 | ,? | This part of the regular expression may cause exponential backtracking on strings starting with '{[A(A)A: ' and containing many repetitions of ',A: '. | | tst.js:197:15:197:16 | a+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'a'. | @@ -152,18 +152,18 @@ | tst.js:254:27:254:29 | \\w* | This part of the regular expression may cause exponential backtracking on strings starting with 'foobarbaz' and containing many repetitions of 'foobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbaz'. | | tst.js:254:39:254:41 | \\w* | This part of the regular expression may cause exponential backtracking on strings starting with 'foobarbazfoobarbaz' and containing many repetitions of 'foobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbaz'. | | tst.js:254:51:254:53 | \\w* | This part of the regular expression may cause exponential backtracking on strings starting with 'foobarbazfoobarbazfoobarbaz' and containing many repetitions of 'foobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbaz'. | -| tst.js:257:14:257:116 | (.thisisagoddamnlongstringforstresstestingthequery\|\\sthisisagoddamnlongstringforstresstestingthequery)* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of ' thisisagoddamnlongstringforstresstestingthequery'. | +| tst.js:257:14:257:116 | (.thisisagoddamnlongstringforstresstestingthequery\|\\sthisisagoddamnlongstringforstresstestingthequery)* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '\\tthisisagoddamnlongstringforstresstestingthequery'. | | tst.js:260:14:260:77 | (thisisagoddamnlongstringforstresstestingthequery\|this\\w+query)* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'thisisagoddamnlongstringforstresstestingthequery'. | -| tst.js:260:68:260:70 | \\w+ | This part of the regular expression may cause exponential backtracking on strings starting with 'this' and containing many repetitions of 'aquerythis'. | +| tst.js:260:68:260:70 | \\w+ | This part of the regular expression may cause exponential backtracking on strings starting with 'this' and containing many repetitions of '0querythis'. | | tst.js:272:21:272:22 | b+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'b'. | -| tst.js:275:38:275:40 | \\s* | This part of the regular expression may cause exponential backtracking on strings starting with 'a]+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '='. | | redos.py:190:27:190:29 | \\s* | This part of the regular expression may cause exponential backtracking on strings starting with '\\n' and containing many repetitions of '\\n'. | -| redos.py:193:28:193:30 | \\s+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of ' '. | +| redos.py:193:28:193:30 | \\s+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '\\t'. | | redos.py:196:78:196:89 | [ a-zA-Z{}]+ | This part of the regular expression may cause exponential backtracking on strings starting with '{[A(A)A:' and containing many repetitions of ' A:'. | | redos.py:196:91:196:92 | ,? | This part of the regular expression may cause exponential backtracking on strings starting with '{[A(A)A: ' and containing many repetitions of ',A: '. | | redos.py:199:25:199:26 | a+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'a'. | @@ -65,20 +65,20 @@ | redos.py:256:37:256:39 | \\w* | This part of the regular expression may cause exponential backtracking on strings starting with 'foobarbaz' and containing many repetitions of 'foobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbaz'. | | redos.py:256:49:256:51 | \\w* | This part of the regular expression may cause exponential backtracking on strings starting with 'foobarbazfoobarbaz' and containing many repetitions of 'foobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbaz'. | | redos.py:256:61:256:63 | \\w* | This part of the regular expression may cause exponential backtracking on strings starting with 'foobarbazfoobarbazfoobarbaz' and containing many repetitions of 'foobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbaz'. | -| redos.py:259:24:259:126 | (.thisisagoddamnlongstringforstresstestingthequery\|\\sthisisagoddamnlongstringforstresstestingthequery)* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of ' thisisagoddamnlongstringforstresstestingthequery'. | +| redos.py:259:24:259:126 | (.thisisagoddamnlongstringforstresstestingthequery\|\\sthisisagoddamnlongstringforstresstestingthequery)* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '\\tthisisagoddamnlongstringforstresstestingthequery'. | | redos.py:262:24:262:87 | (thisisagoddamnlongstringforstresstestingthequery\|this\\w+query)* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'thisisagoddamnlongstringforstresstestingthequery'. | -| redos.py:262:78:262:80 | \\w+ | This part of the regular expression may cause exponential backtracking on strings starting with 'this' and containing many repetitions of 'aquerythis'. | +| redos.py:262:78:262:80 | \\w+ | This part of the regular expression may cause exponential backtracking on strings starting with 'this' and containing many repetitions of '0querythis'. | | redos.py:268:28:268:39 | ([\ufffd\ufffd]\|[\ufffd\ufffd])* | This part of the regular expression may cause exponential backtracking on strings starting with 'foo' and containing many repetitions of '\ufffd'. | | redos.py:271:28:271:41 | ((\ufffd\|\ufffd)\|(\ufffd\|\ufffd))* | This part of the regular expression may cause exponential backtracking on strings starting with 'foo' and containing many repetitions of '\ufffd'. | | redos.py:274:31:274:32 | b+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'b'. | -| redos.py:277:48:277:50 | \\s* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '"" a='. | +| redos.py:277:48:277:50 | \\s* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '""\\t0='. | | redos.py:283:26:283:27 | a+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'a'. | | redos.py:286:26:286:27 | a+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'a'. | | redos.py:292:26:292:27 | a+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'a'. | | redos.py:295:35:295:36 | a+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'a'. | | redos.py:301:100:301:101 | e+ | This part of the regular expression may cause exponential backtracking on strings starting with ';00000000000000' and containing many repetitions of 'e'. | | redos.py:304:28:304:29 | c+ | This part of the regular expression may cause exponential backtracking on strings starting with 'ab' and containing many repetitions of 'c'. | -| redos.py:307:28:307:30 | \\s+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of ' '. | +| redos.py:307:28:307:30 | \\s+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '\\t'. | | redos.py:310:26:310:34 | ([^/]\|X)+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'X'. | | redos.py:313:30:313:34 | [^Y]+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'Xx'. | | redos.py:316:25:316:26 | a* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'a'. | diff --git a/ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtil.qll b/ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtil.qll index 35f8ffc0e65..58900594158 100644 --- a/ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtil.qll +++ b/ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtil.qll @@ -199,7 +199,7 @@ CharClass getCanonicalCharClass(RegExpTerm term) { /** * Holds if `a` and `b` are input symbols from the same regexp. */ -private predicate sharesRoot(TInputSymbol a, TInputSymbol b) { +private predicate sharesRoot(InputSymbol a, InputSymbol b) { exists(RegExpRoot root | belongsTo(a, root) and belongsTo(b, root) @@ -209,7 +209,7 @@ private predicate sharesRoot(TInputSymbol a, TInputSymbol b) { /** * Holds if the `a` is an input symbol from a regexp that has root `root`. */ -private predicate belongsTo(TInputSymbol a, RegExpRoot root) { +private predicate belongsTo(InputSymbol a, RegExpRoot root) { exists(State s | getRoot(s.getRepr()) = root | delta(s, a, _) or @@ -378,6 +378,13 @@ private module CharacterClasses { ) } + bindingset[char, cc] + private string caseNormalize(string char, RegExpTerm cc) { + if RegExpFlags::isIgnoreCase(cc.getRootTerm()) + then result = char.toLowerCase() + else result = char + } + /** * An implementation of `CharacterClass` for positive (non inverted) character classes. */ @@ -386,7 +393,7 @@ private module CharacterClasses { PositiveCharacterClass() { this = getCanonicalCharClass(cc) and not cc.isInverted() } - override string getARelevantChar() { result = getAMentionedChar(cc) } + override string getARelevantChar() { result = caseNormalize(getAMentionedChar(cc), cc) } override predicate matches(string char) { hasChildThatMatches(cc, char) } } @@ -400,8 +407,8 @@ private module CharacterClasses { InvertedCharacterClass() { this = getCanonicalCharClass(cc) and cc.isInverted() } override string getARelevantChar() { - result = nextChar(getAMentionedChar(cc)) or - nextChar(result) = getAMentionedChar(cc) + result = nextChar(caseNormalize(getAMentionedChar(cc), cc)) or + nextChar(result) = caseNormalize(getAMentionedChar(cc), cc) } bindingset[char] @@ -428,13 +435,12 @@ private module CharacterClasses { */ private class PositiveCharacterClassEscape extends CharacterClass { string charClass; + RegExpTerm cc; PositiveCharacterClassEscape() { - exists(RegExpTerm cc | - isEscapeClass(cc, charClass) and - this = getCanonicalCharClass(cc) and - charClass = ["d", "s", "w"] - ) + isEscapeClass(cc, charClass) and + this = getCanonicalCharClass(cc) and + charClass = ["d", "s", "w"] } override string getARelevantChar() { @@ -445,7 +451,9 @@ private module CharacterClasses { result = " " or charClass = "w" and - result = ["a", "Z", "_", "0", "9"] + if RegExpFlags::isIgnoreCase(cc.getRootTerm()) + then result = ["a", "z", "_", "0", "9"] + else result = ["a", "Z", "_", "0", "9"] } override predicate matches(string char) { classEscapeMatches(charClass, char) } @@ -492,6 +500,34 @@ private module CharacterClasses { not classEscapeMatches(charClass.toLowerCase(), char) } } + + /** Gets a representative for all char classes that match the same chars as `c`. */ + CharacterClass normalize(CharacterClass c) { + exists(string normalization | + normalization = getMormalizationString(c) and + result = + min(CharacterClass cc, string raw | + getMormalizationString(cc) = normalization and cc = CharClass(raw) + | + cc order by raw + ) + ) + } + + /** Gets a string representing all the chars matched by `c` */ + private string getMormalizationString(CharacterClass c) { + (c instanceof PositiveCharacterClass or c instanceof PositiveCharacterClassEscape) and + result = concat(string char | c.matches(char) and char = CharacterClasses::getARelevantChar()) + or + (c instanceof InvertedCharacterClass or c instanceof NegativeCharacterClassEscape) and + // the string produced by the concat can not contain repeated chars + // so by starting the below with "nn" we can guarantee that + // it will not overlap with the above case. + // and a negative char class can never match the same chars as a positive one, so we don't miss any results from this. + result = + "nn:" + + concat(string char | not c.matches(char) and char = CharacterClasses::getARelevantChar()) + } } private class EdgeLabel extends TInputSymbol { @@ -620,13 +656,17 @@ predicate delta(State q1, EdgeLabel lbl, State q2) { cc.isUniversalClass() and q1 = before(cc) and lbl = Any() and q2 = after(cc) or q1 = before(cc) and - lbl = CharClass(cc.getRawValue() + "|" + getCanonicalizationFlags(cc.getRootTerm())) and + lbl = + CharacterClasses::normalize(CharClass(cc.getRawValue() + "|" + + getCanonicalizationFlags(cc.getRootTerm()))) and q2 = after(cc) ) or exists(RegExpTerm cc | isEscapeClass(cc, _) | q1 = before(cc) and - lbl = CharClass(cc.getRawValue() + "|" + getCanonicalizationFlags(cc.getRootTerm())) and + lbl = + CharacterClasses::normalize(CharClass(cc.getRawValue() + "|" + + getCanonicalizationFlags(cc.getRootTerm()))) and q2 = after(cc) ) or diff --git a/ruby/ql/test/query-tests/security/cwe-1333-exponential-redos/ReDoS.expected b/ruby/ql/test/query-tests/security/cwe-1333-exponential-redos/ReDoS.expected index 198b362184c..11212e3d806 100644 --- a/ruby/ql/test/query-tests/security/cwe-1333-exponential-redos/ReDoS.expected +++ b/ruby/ql/test/query-tests/security/cwe-1333-exponential-redos/ReDoS.expected @@ -29,7 +29,7 @@ | tst.rb:125:11:125:24 | ([a-z]\|[d-h])* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'd'. | | tst.rb:128:11:128:26 | ([^a-z]\|[^0-9])* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '/'. | | tst.rb:131:11:131:21 | (\\d\|[0-9])* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '0'. | -| tst.rb:134:11:134:18 | (\\s\|\\s)* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of ' '. | +| tst.rb:134:11:134:18 | (\\s\|\\s)* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '\\t'. | | tst.rb:137:11:137:17 | (\\w\|G)* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'G'. | | tst.rb:143:11:143:18 | (\\d\|\\w)* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '0'. | | tst.rb:146:11:146:17 | (\\d\|5)* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '5'. | @@ -44,7 +44,7 @@ | tst.rb:173:12:173:16 | [\\d]+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '0'. | | tst.rb:185:12:185:17 | [^>a]+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '='. | | tst.rb:188:13:188:15 | \\s* | This part of the regular expression may cause exponential backtracking on strings starting with '\\n' and containing many repetitions of '\\n'. | -| tst.rb:191:14:191:16 | \\s+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of ' '. | +| tst.rb:191:14:191:16 | \\s+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '\\t'. | | tst.rb:194:64:194:75 | [ a-zA-Z{}]+ | This part of the regular expression may cause exponential backtracking on strings starting with '{[A(A)A:' and containing many repetitions of ' A:'. | | tst.rb:194:77:194:78 | ,? | This part of the regular expression may cause exponential backtracking on strings starting with '{[A(A)A: ' and containing many repetitions of ',A: '. | | tst.rb:197:11:197:12 | a+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'a'. | @@ -63,18 +63,18 @@ | tst.rb:254:23:254:25 | \\w* | This part of the regular expression may cause exponential backtracking on strings starting with 'foobarbaz' and containing many repetitions of 'foobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbaz'. | | tst.rb:254:35:254:37 | \\w* | This part of the regular expression may cause exponential backtracking on strings starting with 'foobarbazfoobarbaz' and containing many repetitions of 'foobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbaz'. | | tst.rb:254:47:254:49 | \\w* | This part of the regular expression may cause exponential backtracking on strings starting with 'foobarbazfoobarbazfoobarbaz' and containing many repetitions of 'foobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbaz'. | -| tst.rb:257:10:257:112 | (.thisisagoddamnlongstringforstresstestingthequery\|\\sthisisagoddamnlongstringforstresstestingthequery)* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of ' thisisagoddamnlongstringforstresstestingthequery'. | +| tst.rb:257:10:257:112 | (.thisisagoddamnlongstringforstresstestingthequery\|\\sthisisagoddamnlongstringforstresstestingthequery)* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '\\tthisisagoddamnlongstringforstresstestingthequery'. | | tst.rb:260:10:260:73 | (thisisagoddamnlongstringforstresstestingthequery\|this\\w+query)* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'thisisagoddamnlongstringforstresstestingthequery'. | -| tst.rb:260:64:260:66 | \\w+ | This part of the regular expression may cause exponential backtracking on strings starting with 'this' and containing many repetitions of 'aquerythis'. | +| tst.rb:260:64:260:66 | \\w+ | This part of the regular expression may cause exponential backtracking on strings starting with 'this' and containing many repetitions of '0querythis'. | | tst.rb:272:17:272:18 | b+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'b'. | -| tst.rb:275:34:275:36 | \\s* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '"" a='. | +| tst.rb:275:34:275:36 | \\s* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '""\\t0='. | | tst.rb:281:12:281:13 | a+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'a'. | | tst.rb:284:12:284:13 | a+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'a'. | | tst.rb:290:12:290:13 | a+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'a'. | | tst.rb:293:21:293:22 | a+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'a'. | | tst.rb:299:86:299:87 | e+ | This part of the regular expression may cause exponential backtracking on strings starting with ';00000000000000' and containing many repetitions of 'e'. | | tst.rb:302:14:302:15 | c+ | This part of the regular expression may cause exponential backtracking on strings starting with 'ab' and containing many repetitions of 'c'. | -| tst.rb:305:14:305:16 | \\s+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of ' '. | +| tst.rb:305:14:305:16 | \\s+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '\\t'. | | tst.rb:308:12:308:21 | ([^\\/]\|X)+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'X'. | | tst.rb:311:16:311:20 | [^Y]+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'Xx'. | | tst.rb:314:11:314:12 | a* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'a'. | From dbeae9aefbc3431637693cae0b459a58dbe2be30 Mon Sep 17 00:00:00 2001 From: Erik Krogh Kristensen Date: Tue, 22 Mar 2022 11:42:11 +0100 Subject: [PATCH 004/600] make a parameterized module out of the RegexpMatching implementation --- config/identical-files.json | 5 + .../javascript/security/BadTagFilterQuery.qll | 222 +++--------------- .../security/performance/RegexpMatching.qll | 175 ++++++++++++++ .../python/security/BadTagFilterQuery.qll | 222 +++--------------- .../security/performance/RegexpMatching.qll | 175 ++++++++++++++ .../ruby/security/BadTagFilterQuery.qll | 222 +++--------------- .../security/performance/RegexpMatching.qll | 175 ++++++++++++++ 7 files changed, 626 insertions(+), 570 deletions(-) create mode 100644 javascript/ql/lib/semmle/javascript/security/performance/RegexpMatching.qll create mode 100644 python/ql/lib/semmle/python/security/performance/RegexpMatching.qll create mode 100644 ruby/ql/lib/codeql/ruby/security/performance/RegexpMatching.qll diff --git a/config/identical-files.json b/config/identical-files.json index 77e39399cec..0c319a636c8 100644 --- a/config/identical-files.json +++ b/config/identical-files.json @@ -501,6 +501,11 @@ "ruby/ql/lib/codeql/ruby/security/performance/SuperlinearBackTracking.qll", "java/ql/lib/semmle/code/java/security/performance/SuperlinearBackTracking.qll" ], + "RegexpMatching Python/JS/Ruby": [ + "javascript/ql/lib/semmle/javascript/security/performance/RegexpMatching.qll", + "python/ql/lib/semmle/python/security/performance/RegexpMatching.qll", + "ruby/ql/lib/codeql/ruby/security/performance/RegexpMatching.qll" + ], "BadTagFilterQuery Python/JS/Ruby": [ "javascript/ql/lib/semmle/javascript/security/BadTagFilterQuery.qll", "python/ql/lib/semmle/python/security/BadTagFilterQuery.qll", diff --git a/javascript/ql/lib/semmle/javascript/security/BadTagFilterQuery.qll b/javascript/ql/lib/semmle/javascript/security/BadTagFilterQuery.qll index d935abce37a..6b3c230e7e8 100644 --- a/javascript/ql/lib/semmle/javascript/security/BadTagFilterQuery.qll +++ b/javascript/ql/lib/semmle/javascript/security/BadTagFilterQuery.qll @@ -2,196 +2,25 @@ * Provides precicates for reasoning about bad tag filter vulnerabilities. */ -import performance.ReDoSUtil +import performance.RegexpMatching /** - * A module for determining if a regexp matches a given string, - * and reasoning about which capture groups are filled by a given string. + * Holds if the regexp `root` should be tested against `str`. + * Implements the `isRegexpMatchingCandidateSig` signature from `RegexpMatching`. */ -private module RegexpMatching { - /** - * A class to test whether a regular expression matches a string. - * Override this class and extend `test`/`testWithGroups` to configure which strings should be tested for acceptance by this regular expression. - * The result can afterwards be read from the `matches` predicate. - * - * Strings in the `testWithGroups` predicate are also tested for which capture groups are filled by the given string. - * The result is available in the `fillCaptureGroup` predicate. - */ - abstract class MatchedRegExp extends RegExpTerm { - MatchedRegExp() { this.isRootTerm() } - - /** - * Holds if it should be tested whether this regular expression matches `str`. - * - * If `ignorePrefix` is true, then a regexp without a start anchor will be treated as if it had a start anchor. - * E.g. a regular expression `/foo$/` will match any string that ends with "foo", - * but if `ignorePrefix` is true, it will only match "foo". - */ - predicate test(string str, boolean ignorePrefix) { - none() // maybe overridden in subclasses - } - - /** - * Same as `test(..)`, but where the `fillsCaptureGroup` afterwards tells which capture groups were filled by the given string. - */ - predicate testWithGroups(string str, boolean ignorePrefix) { - none() // maybe overridden in subclasses - } - - /** - * Holds if this RegExp matches `str`, where `str` is either in the `test` or `testWithGroups` predicate. - */ - final predicate matches(string str) { - exists(State state | state = getAState(this, str.length() - 1, str, _) | - epsilonSucc*(state) = Accept(_) - ) - } - - /** - * Holds if matching `str` may fill capture group number `g`. - * Only holds if `str` is in the `testWithGroups` predicate. - */ - final predicate fillsCaptureGroup(string str, int g) { - exists(State s | - s = getAStateThatReachesAccept(this, _, str, _) and - g = group(s.getRepr()) - ) - } - } - - /** - * Gets a state the regular expression `reg` can be in after matching the `i`th char in `str`. - * The regular expression is modeled as a non-determistic finite automaton, - * the regular expression can therefore be in multiple states after matching a character. - * - * It's a forward search to all possible states, and there is thus no guarantee that the state is on a path to an accepting state. - */ - private State getAState(MatchedRegExp reg, int i, string str, boolean ignorePrefix) { - // start state, the -1 position before any chars have been matched - i = -1 and - ( - reg.test(str, ignorePrefix) - or - reg.testWithGroups(str, ignorePrefix) - ) and - result.getRepr().getRootTerm() = reg and - isStartState(result) +private predicate isBadTagFilterCandidate( + RootTerm root, string str, boolean ignorePrefix, boolean testWithGroups +) { + // the regexp must mention "<" and ">" explicitly. + forall(string angleBracket | angleBracket = ["<", ">"] | + any(RegExpConstant term | term.getValue().matches("%" + angleBracket + "%")).getRootTerm() = + root + ) and + ignorePrefix = true and + ( + str = ["", "", "", "", "", @@ -200,12 +29,25 @@ class HtmlMatchingRegExp extends RegexpMatching::MatchedRegExp { "", "", "", "", "", "", "", "", @@ -200,12 +29,25 @@ class HtmlMatchingRegExp extends RegexpMatching::MatchedRegExp { "", "", "", "", "", "", "", "", @@ -200,12 +29,25 @@ class HtmlMatchingRegExp extends RegexpMatching::MatchedRegExp { "", "", "", "", "", "", "", "