From a27142d2d4c8fe6118ffe41ece771caca9e26427 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 15 Jul 2026 21:45:03 +0000 Subject: [PATCH] Add C++ regex parser and RegexTreeView (Phase 1) --- .../2026-07-15-cpp-regex-tree-view.md | 4 + cpp/ql/lib/qlpack.yml | 1 + .../semmle/code/cpp/regex/RegexTreeView.qll | 947 ++++++++++++++++++ .../code/cpp/regex/internal/ParseRegExp.qll | 840 ++++++++++++++++ .../library-tests/regex/CompileCheck.expected | 0 .../test/library-tests/regex/CompileCheck.ql | 24 + .../library-tests/regex/Consistency.expected | 0 .../test/library-tests/regex/Consistency.ql | 13 + .../test/library-tests/regex/parse.expected | 0 cpp/ql/test/library-tests/regex/parse.ql | 27 + .../test/library-tests/regex/regexp.expected | 0 cpp/ql/test/library-tests/regex/regexp.ql | 10 + cpp/ql/test/library-tests/regex/test.cpp | 83 ++ 13 files changed, 1949 insertions(+) create mode 100644 cpp/ql/lib/change-notes/2026-07-15-cpp-regex-tree-view.md create mode 100644 cpp/ql/lib/semmle/code/cpp/regex/RegexTreeView.qll create mode 100644 cpp/ql/lib/semmle/code/cpp/regex/internal/ParseRegExp.qll create mode 100644 cpp/ql/test/library-tests/regex/CompileCheck.expected create mode 100644 cpp/ql/test/library-tests/regex/CompileCheck.ql create mode 100644 cpp/ql/test/library-tests/regex/Consistency.expected create mode 100644 cpp/ql/test/library-tests/regex/Consistency.ql create mode 100644 cpp/ql/test/library-tests/regex/parse.expected create mode 100644 cpp/ql/test/library-tests/regex/parse.ql create mode 100644 cpp/ql/test/library-tests/regex/regexp.expected create mode 100644 cpp/ql/test/library-tests/regex/regexp.ql create mode 100644 cpp/ql/test/library-tests/regex/test.cpp diff --git a/cpp/ql/lib/change-notes/2026-07-15-cpp-regex-tree-view.md b/cpp/ql/lib/change-notes/2026-07-15-cpp-regex-tree-view.md new file mode 100644 index 00000000000..55e7b109192 --- /dev/null +++ b/cpp/ql/lib/change-notes/2026-07-15-cpp-regex-tree-view.md @@ -0,0 +1,4 @@ +--- +category: newQuery +--- +* Added an internal C++ regular-expression parse library (`semmle.code.cpp.regex.RegexTreeView`) that implements the shared `RegexTreeViewSig` signature. This is the foundation for future ReDoS analyses for C++ (`cpp/polynomial-redos` and `cpp/redos`). No new queries are added in this release. diff --git a/cpp/ql/lib/qlpack.yml b/cpp/ql/lib/qlpack.yml index 04f66548112..d7c4c6a8a39 100644 --- a/cpp/ql/lib/qlpack.yml +++ b/cpp/ql/lib/qlpack.yml @@ -10,6 +10,7 @@ dependencies: codeql/mad: ${workspace} codeql/quantum: ${workspace} codeql/rangeanalysis: ${workspace} + codeql/regex: ${workspace} codeql/ssa: ${workspace} codeql/typeflow: ${workspace} codeql/tutorial: ${workspace} diff --git a/cpp/ql/lib/semmle/code/cpp/regex/RegexTreeView.qll b/cpp/ql/lib/semmle/code/cpp/regex/RegexTreeView.qll new file mode 100644 index 00000000000..e0b6d3347a2 --- /dev/null +++ b/cpp/ql/lib/semmle/code/cpp/regex/RegexTreeView.qll @@ -0,0 +1,947 @@ +/** + * Provides a class hierarchy corresponding to a parse tree of C++ regular + * expressions, and an implementation of `RegexTreeViewSig` for use with the + * shared ReDoS analysis libraries. + * + * The target dialect is **ECMAScript** (the default for `std::regex`). + * + * Usage: + * ```ql + * import semmle.code.cpp.regex.RegexTreeView + * // RegExpTerm, RegExpGroup, etc. are available directly. + * ``` + */ + +import cpp +private import semmle.code.cpp.regex.internal.ParseRegExp +private import codeql.regex.RegexTreeView +// Export the implementation both as `RegexTreeView` (for use as a functor +// argument) and in the top-level scope (for direct import). +import Impl as RegexTreeView +import Impl + +/** Gets the root parse-tree term for `re`, if it has been parsed. */ +RegExpTerm getParsedRegExp(StringLiteral re) { result.getRegex() = re and result.isRootTerm() } + +// --------------------------------------------------------------------------- +// newtype TRegExpParent +// --------------------------------------------------------------------------- + +/** + * An element that is either a regex literal (the root of a parse tree) or a + * regex term (a node in the parse tree). + * + * For sequences and alternations we require at least one child; otherwise the + * node is represented as its sole element instead. + */ +private newtype TRegExpParent = + /** A string literal used as a regular expression. */ + TRegExpLiteral(RegExp re) or + /** A quantified term. */ + TRegExpQuantifier(RegExp re, int start, int end) { re.qualifiedItem(start, end, _, _) } or + /** A sequence of two or more items. */ + TRegExpSequence(RegExp re, int start, int end) { + re.sequence(start, end) and + exists(seqChild(re, start, end, 1)) // require at least two children + } or + /** An alternation (`a|b`). */ + TRegExpAlt(RegExp re, int start, int end) { + re.alternation(start, end) and + exists(int part_end | + re.alternationOption(start, end, start, part_end) and part_end < end + ) // require at least two alternatives + } or + /** A character class (`[...]`). */ + TRegExpCharacterClass(RegExp re, int start, int end) { re.charSet(start, end) } or + /** A character range inside a character class (`a-z`). */ + TRegExpCharacterRange(RegExp re, int start, int end) { re.charRange(_, start, _, _, end) } or + /** A group (`(...)`, `(?:...)`, `(?...)`, etc.). */ + TRegExpGroup(RegExp re, int start, int end) { re.group(start, end) } or + /** A special (meta) character (`.`, `^`, `$`, `\b`, `\B`). */ + TRegExpSpecialChar(RegExp re, int start, int end) { re.specialCharacter(start, end, _) } or + /** A normal character or escape sequence (not a special character). */ + TRegExpNormalChar(RegExp re, int start, int end) { + re.normalCharacterSequence(start, end) + or + re.escapedCharacter(start, end) and + not re.specialCharacter(start, end, _) + } or + /** A back-reference (`\1`, `\k`). */ + TRegExpBackRef(RegExp re, int start, int end) { re.backreference(start, end) } + +// --------------------------------------------------------------------------- +// Helper: sequence children +// --------------------------------------------------------------------------- + +pragma[nomagic] +private int seqChildEnd(RegExp re, int start, int end, int i) { + result = seqChild(re, start, end, i).getEnd() +} + +private RegExpTerm seqChild(RegExp re, int start, int end, int i) { + re.sequence(start, end) and + ( + i = 0 and + result.getRegex() = re and + result.getStart() = start and + exists(int itemEnd | + re.item(start, itemEnd) and + result.getEnd() = itemEnd + ) + or + i > 0 and + result.getRegex() = re and + exists(int itemStart | itemStart = seqChildEnd(re, start, end, i - 1) | + result.getStart() = itemStart and + re.item(itemStart, result.getEnd()) + ) + ) +} + +// --------------------------------------------------------------------------- +// Module Impl (implements RegexTreeViewSig) +// --------------------------------------------------------------------------- + +/** An implementation that satisfies the `RegexTreeViewSig` signature. */ +module Impl implements RegexTreeViewSig { + // ------------------------------------------------------------------------- + // RegExpParent + // ------------------------------------------------------------------------- + + /** + * An element containing a regular expression term: either the literal itself + * or a term node. + */ + class RegExpParent extends TRegExpParent { + /** Gets a textual representation of this element. */ + string toString() { result = "RegExpParent" } + + /** Gets the `i`th child term. */ + abstract RegExpTerm getChild(int i); + + /** Gets any child term. */ + RegExpTerm getAChild() { result = this.getChild(_) } + + /** Gets the number of child terms. */ + int getNumChild() { result = count(this.getAChild()) } + + /** Gets the last child term. */ + RegExpTerm getLastChild() { result = this.getChild(this.getNumChild() - 1) } + + /** Gets the underlying regex. */ + abstract RegExp getRegex(); + } + + // ------------------------------------------------------------------------- + // RegExpLiteral + // ------------------------------------------------------------------------- + + /** A string literal used as a regular expression. */ + class RegExpLiteral extends TRegExpLiteral, RegExpParent { + RegExp re; + + RegExpLiteral() { this = TRegExpLiteral(re) } + + override RegExpTerm getChild(int i) { + i = 0 and result.getRegex() = re and result.isRootTerm() + } + + /** + * Holds if dot `.` matches all characters including newlines. + * + * TODO (Phase 2): Implement by detecting `std::regex::flag_type::multiline` + * or `dotall` via construction-site dataflow. Conservatively returns nothing. + */ + predicate isDotAll() { none() } + + /** + * Holds if matching is case-insensitive. + * + * TODO (Phase 2): Implement by detecting `std::regex_constants::icase` + * via construction-site dataflow. Conservatively returns nothing. + */ + predicate isIgnoreCase() { none() } + + /** Gets a string representing all flags for this regex. */ + string getFlags() { none() } + + override RegExp getRegex() { result = re } + + /** Gets the primary QL class for this element. */ + string getPrimaryQLClass() { result = "RegExpLiteral" } + } + + // ------------------------------------------------------------------------- + // RegExpTerm (base class for all parse-tree nodes) + // ------------------------------------------------------------------------- + + /** + * A regular expression term — a node in the parse tree of a regex literal. + */ + class RegExpTerm extends RegExpParent { + RegExp re; + int start; + int end; + + RegExpTerm() { + this = TRegExpAlt(re, start, end) + or + this = TRegExpBackRef(re, start, end) + or + this = TRegExpCharacterClass(re, start, end) + or + this = TRegExpCharacterRange(re, start, end) + or + this = TRegExpNormalChar(re, start, end) + or + this = TRegExpGroup(re, start, end) + or + this = TRegExpQuantifier(re, start, end) + or + this = TRegExpSequence(re, start, end) + or + this = TRegExpSpecialChar(re, start, end) + } + + /** Gets the outermost (root) term of this regular expression. */ + RegExpTerm getRootTerm() { + this.isRootTerm() and result = this + or + result = this.getParent().(RegExpTerm).getRootTerm() + } + + /** + * Holds if this term is part of a string literal that is (potentially) + * used as a regular expression. + */ + predicate isUsedAsRegExp() { any() } + + /** + * Holds if this is the root term of a regular expression (i.e., it covers + * the full string value). + */ + predicate isRootTerm() { start = 0 and end = re.getText().length() } + + override RegExpTerm getChild(int i) { + result = this.(RegExpAlt).getChild(i) + or + result = this.(RegExpBackRef).getChild(i) + or + result = this.(RegExpCharacterClass).getChild(i) + or + result = this.(RegExpCharacterRange).getChild(i) + or + result = this.(RegExpNormalChar).getChild(i) + or + result = this.(RegExpGroup).getChild(i) + or + result = this.(RegExpQuantifier).getChild(i) + or + result = this.(RegExpSequence).getChild(i) + or + result = this.(RegExpSpecialChar).getChild(i) + } + + /** + * Gets the parent term (or the literal if this is the root term). + */ + RegExpParent getParent() { result.getAChild() = this } + + override RegExp getRegex() { result = re } + + /** Gets the start offset of this term in the string value. */ + int getStart() { result = start } + + /** Gets the end offset (exclusive) of this term in the string value. */ + int getEnd() { result = end } + + override string toString() { result = re.getText().substring(start, end) } + + /** + * Gets the source location of the enclosing string literal. + * Use `hasLocationInfo` to get offsets within the string. + */ + Location getLocation() { result = re.getLocation() } + + /** + * Holds if this term is found at the given source location. + * + * The location maps back to character offsets within the C++ string literal. + * Because the literal's start column already accounts for the opening `"`, + * we add 1 to skip the quote itself. + */ + predicate hasLocationInfo( + string filepath, int startline, int startcolumn, int endline, int endcolumn + ) { + exists(Location loc | loc = re.getLocation() | + loc.hasLocationInfo(filepath, startline, _, _, _) and + startcolumn = loc.getStartColumn() + start + 1 and + endline = startline and + endcolumn = loc.getStartColumn() + end // end is exclusive, so -1+1 = 0 offset + ) + } + + /** Gets the file this term is in. */ + File getFile() { result = re.getFile() } + + /** Gets the raw source text of this term. */ + string getRawValue() { result = this.toString() } + + /** Gets the string literal enclosing this term. */ + RegExpLiteral getLiteral() { result = TRegExpLiteral(re) } + + /** Gets the term matched (textually) immediately before this one, if any. */ + RegExpTerm getPredecessor() { + exists(RegExpTerm parent | parent = this.getParent() | + result = parent.(RegExpSequence).previousElement(this) + or + not exists(parent.(RegExpSequence).previousElement(this)) and + not parent instanceof RegExpSubPattern and + result = parent.getPredecessor() + ) + } + + /** Gets the term matched (textually) immediately after this one, if any. */ + RegExpTerm getSuccessor() { + exists(RegExpTerm parent | parent = this.getParent() | + result = parent.(RegExpSequence).nextElement(this) + or + not exists(parent.(RegExpSequence).nextElement(this)) and + not parent instanceof RegExpSubPattern and + result = parent.getSuccessor() + ) + } + + /** Gets the primary QL class for this term. */ + string getPrimaryQLClass() { result = "RegExpTerm" } + } + + // ------------------------------------------------------------------------- + // Quantifiers + // ------------------------------------------------------------------------- + + /** + * A quantified regular expression term (`a*`, `a+`, `a?`, `a{n,m}`, etc.). + */ + class RegExpQuantifier extends RegExpTerm, TRegExpQuantifier { + int part_end; + boolean may_repeat_forever; + + RegExpQuantifier() { + this = TRegExpQuantifier(re, start, end) and + re.qualifiedPart(start, part_end, end, _, may_repeat_forever) + } + + override RegExpTerm getChild(int i) { + i = 0 and + result.getRegex() = re and + result.getStart() = start and + result.getEnd() = part_end + } + + /** Holds if this quantifier may match an unbounded number of times. */ + predicate mayRepeatForever() { may_repeat_forever = true } + + /** Gets the textual qualifier (e.g., `*`, `+`, `{2,5}`). */ + string getQualifier() { result = re.getText().substring(part_end, end) } + + override string getPrimaryQLClass() { result = "RegExpQuantifier" } + } + + /** + * A quantifier that permits unlimited repetitions (`*`, `+`, or `{n,}`). + */ + class InfiniteRepetitionQuantifier extends RegExpQuantifier { + InfiniteRepetitionQuantifier() { this.mayRepeatForever() } + } + + /** A star-quantified term (`a*`). */ + class RegExpStar extends InfiniteRepetitionQuantifier { + RegExpStar() { this.getQualifier().charAt(0) = "*" } + + override string getPrimaryQLClass() { result = "RegExpStar" } + } + + /** A plus-quantified term (`a+`). */ + class RegExpPlus extends InfiniteRepetitionQuantifier { + RegExpPlus() { this.getQualifier().charAt(0) = "+" } + + override string getPrimaryQLClass() { result = "RegExpPlus" } + } + + /** An optional term (`a?`). */ + class RegExpOpt extends RegExpQuantifier { + RegExpOpt() { this.getQualifier().charAt(0) = "?" } + + override string getPrimaryQLClass() { result = "RegExpOpt" } + } + + /** + * A range-quantified term (`a{2}`, `a{2,5}`, `a{2,}`). + */ + class RegExpRange extends RegExpQuantifier { + string upper; + string lower; + + RegExpRange() { re.multiples(part_end, end, lower, upper) } + + /** Gets the lower bound of this range. */ + int getLowerBound() { result = lower.toInt() } + + /** + * Gets the upper bound of this range, if any. + * + * For `{n}`, both lower and upper are `n`. + * For `{n,}`, there is no upper bound (this predicate has no result). + */ + int getUpperBound() { result = upper.toInt() } + + override string getPrimaryQLClass() { result = "RegExpRange" } + } + + // ------------------------------------------------------------------------- + // Sequences and alternations + // ------------------------------------------------------------------------- + + /** + * A sequence term — two or more items in a row. + * + * Example: `ab` is a sequence of `a` and `b`. + */ + class RegExpSequence extends RegExpTerm, TRegExpSequence { + RegExpSequence() { this = TRegExpSequence(re, start, end) } + + override RegExpTerm getChild(int i) { result = seqChild(re, start, end, i) } + + /** Gets the element preceding `element` in this sequence. */ + RegExpTerm previousElement(RegExpTerm element) { element = this.nextElement(result) } + + /** Gets the element following `element` in this sequence. */ + RegExpTerm nextElement(RegExpTerm element) { + exists(int i | + element = this.getChild(i) and + result = this.getChild(i + 1) + ) + } + + override string getPrimaryQLClass() { result = "RegExpSequence" } + } + + /** + * An alternation term (`a|b`). + */ + class RegExpAlt extends RegExpTerm, TRegExpAlt { + RegExpAlt() { this = TRegExpAlt(re, start, end) } + + override RegExpTerm getChild(int i) { + i = 0 and + result.getRegex() = re and + result.getStart() = start and + exists(int part_end | + re.alternationOption(start, end, start, part_end) and + result.getEnd() = part_end + ) + or + i > 0 and + result.getRegex() = re and + exists(int part_start | + part_start = this.getChild(i - 1).getEnd() + 1 // skip the `|` + | + result.getStart() = part_start and + re.alternationOption(start, end, part_start, result.getEnd()) + ) + } + + override string getPrimaryQLClass() { result = "RegExpAlt" } + } + + // ------------------------------------------------------------------------- + // Character escapes and normal characters + // ------------------------------------------------------------------------- + + /** + * A normal character in a regular expression (including escape sequences). + */ + additional class RegExpNormalChar extends RegExpTerm, TRegExpNormalChar { + RegExpNormalChar() { this = TRegExpNormalChar(re, start, end) } + + /** Holds if this is a valid Unicode character (always true here). */ + predicate isCharacter() { any() } + + /** Gets the string representation of this character. */ + string getValue() { result = re.getText().substring(start, end) } + + override RegExpTerm getChild(int i) { none() } + + override string getPrimaryQLClass() { result = "RegExpNormalChar" } + } + + /** + * A character escape in a regular expression (`\.`, `\n`, etc.). + * This is a type alias; `RegExpCharEscape` == `RegExpEscape`. + */ + class RegExpCharEscape = RegExpEscape; + + /** + * An escaped regular expression term — starts with `\` and is not a + * back-reference. + */ + class RegExpEscape extends RegExpNormalChar { + RegExpEscape() { re.escapedCharacter(start, end) } + + /** + * Gets the name of the escaped character; for example, `w` for `\w` and + * `n` for `\n`. + */ + override string getValue() { + this.isIdentityEscape() and result = this.getUnescaped() + or + this.getUnescaped() = "n" and result = "\n" + or + this.getUnescaped() = "r" and result = "\r" + or + this.getUnescaped() = "t" and result = "\t" + or + this.getUnescaped() = "f" and result = 12.toUnicode() + or + this.getUnescaped() = "v" and result = 11.toUnicode() + } + + /** Holds if this escape's name is the character following the backslash. */ + predicate isIdentityEscape() { not this.getUnescaped() in ["n", "r", "t", "f", "v"] } + + override string getPrimaryQLClass() { result = "RegExpEscape" } + + /** Gets the part of the term following the backslash (e.g., `w` for `\w`). */ + string getUnescaped() { result = re.getText().substring(start + 1, end) } + } + + /** + * A character-class escape — an escape that denotes a set of characters. + * + * Examples: `\d`, `\D`, `\w`, `\W`, `\s`, `\S`. + */ + class RegExpCharacterClassEscape extends RegExpEscape { + RegExpCharacterClassEscape() { this.getValue() in ["d", "D", "s", "S", "w", "W"] } + + override RegExpTerm getChild(int i) { none() } + + override string getPrimaryQLClass() { result = "RegExpCharacterClassEscape" } + } + + // ------------------------------------------------------------------------- + // Character classes [...] + // ------------------------------------------------------------------------- + + /** + * A character class in a regular expression (`[a-z]`, `[^0-9]`, etc.). + */ + class RegExpCharacterClass extends RegExpTerm, TRegExpCharacterClass { + RegExpCharacterClass() { this = TRegExpCharacterClass(re, start, end) } + + /** Holds if this character class is inverted (`[^...]`). */ + predicate isInverted() { re.getChar(start + 1) = "^" } + + /** + * Holds if this character class matches any character. + * + * That is, `[^]` (empty inverted class) or `[\w\W]`/`[\d\D]`/`[\s\S]` + * (complementary class escapes). + */ + predicate isUniversalClass() { + // [^] — empty inverted class + this.isInverted() and not exists(this.getAChild()) + or + // [\w\W] and similar — two complementary class escapes + not this.isInverted() and + exists(string cce1, string cce2 | + cce1 = this.getAChild().(RegExpCharacterClassEscape).getValue() and + cce2 = this.getAChild().(RegExpCharacterClassEscape).getValue() + | + cce1 != cce2 and cce1.toLowerCase() = cce2.toLowerCase() + ) + } + + override RegExpTerm getChild(int i) { + i = 0 and + result.getRegex() = re and + exists(int itemStart, int itemEnd | + result.getStart() = itemStart and + re.char_set_start(start, itemStart) and + re.char_set_child(start, itemStart, itemEnd) and + result.getEnd() = itemEnd + ) + or + i > 0 and + result.getRegex() = re and + exists(int itemStart | itemStart = this.getChild(i - 1).getEnd() | + result.getStart() = itemStart and + re.char_set_child(start, itemStart, result.getEnd()) + ) + } + + override string getPrimaryQLClass() { result = "RegExpCharacterClass" } + } + + /** + * A character range inside a character class (`a-z`). + */ + class RegExpCharacterRange extends RegExpTerm, TRegExpCharacterRange { + int lower_end; + int upper_start; + + RegExpCharacterRange() { + this = TRegExpCharacterRange(re, start, end) and + re.charRange(_, start, lower_end, upper_start, end) + } + + /** Holds if this range spans from `lo` to `hi`. */ + predicate isRange(string lo, string hi) { + lo = re.getText().substring(start, lower_end) and + hi = re.getText().substring(upper_start, end) + } + + override RegExpTerm getChild(int i) { + i = 0 and + result.getRegex() = re and + result.getStart() = start and + result.getEnd() = lower_end + or + i = 1 and + result.getRegex() = re and + result.getStart() = upper_start and + result.getEnd() = end + } + + override string getPrimaryQLClass() { result = "RegExpCharacterRange" } + } + + // ------------------------------------------------------------------------- + // Special characters (`.`, `^`, `$`, `\b`, `\B`) + // ------------------------------------------------------------------------- + + /** + * A special (meta) character in a regular expression. + * + * In ECMAScript: `.`, `^`, `$`, `\b`, `\B`. + */ + additional class RegExpSpecialChar extends RegExpTerm, TRegExpSpecialChar { + string char; + + RegExpSpecialChar() { + this = TRegExpSpecialChar(re, start, end) and + re.specialCharacter(start, end, char) + } + + /** Holds if this constant is a valid Unicode character (always true). */ + predicate isCharacter() { any() } + + /** Gets the character (e.g., `"."`, `"^"`, `"\\b"`). */ + string getChar() { result = char } + + override RegExpTerm getChild(int i) { none() } + + override string getPrimaryQLClass() { result = "RegExpSpecialChar" } + } + + /** A dot (`.`) that matches any character (except possibly newlines). */ + class RegExpDot extends RegExpSpecialChar { + RegExpDot() { this.getChar() = "." } + + override string getPrimaryQLClass() { result = "RegExpDot" } + } + + /** An anchor term (`^`, `$`). */ + class RegExpAnchor extends RegExpSpecialChar { + RegExpAnchor() { this.getChar() = ["^", "$"] } + + override string getChar() { result = char } + } + + /** + * A dollar assertion `$`, matching the end of the input (or end of a line in + * multiline mode). + */ + class RegExpDollar extends RegExpAnchor { + RegExpDollar() { this.getChar() = "$" } + + override string getPrimaryQLClass() { result = "RegExpDollar" } + } + + /** + * A caret assertion `^`, matching the start of the input (or start of a line + * in multiline mode). + */ + class RegExpCaret extends RegExpAnchor { + RegExpCaret() { this.getChar() = "^" } + + override string getPrimaryQLClass() { result = "RegExpCaret" } + } + + /** A word-boundary assertion `\b`. */ + class RegExpWordBoundary extends RegExpSpecialChar { + RegExpWordBoundary() { this.getChar() = "\\b" } + + override string getPrimaryQLClass() { result = "RegExpWordBoundary" } + } + + /** A non-word-boundary assertion `\B`. */ + class RegExpNonWordBoundary extends RegExpSpecialChar { + RegExpNonWordBoundary() { this.getChar() = "\\B" } + + override string getPrimaryQLClass() { result = "RegExpNonWordBoundary" } + } + + // ------------------------------------------------------------------------- + // Groups (capturing, non-capturing, named, lookahead/lookbehind) + // ------------------------------------------------------------------------- + + /** + * A grouped regular expression: `(...)`, `(?:...)`, `(?...)`, or an + * assertion group `(?=...)`, etc. + */ + class RegExpGroup extends RegExpTerm, TRegExpGroup { + RegExpGroup() { this = TRegExpGroup(re, start, end) } + + /** + * Gets the index of this capture group within the enclosing regex literal. + * + * For example, in `((a?).)(?:b)`: + * - `((a?).)` has index 1 + * - `(a?)` has index 2 + * - `(?:b)` has no index (non-capturing) + */ + int getNumber() { result = re.getGroupNumber(start, end) } + + /** Holds if this is a capture group (has an index). */ + predicate isCapture() { exists(this.getNumber()) } + + /** Holds if this is a named capture group. */ + predicate isNamed() { exists(this.getName()) } + + /** Gets the name of this named capture group, if any. */ + string getName() { result = re.getGroupName(start, end) } + + override RegExpTerm getChild(int i) { + result.getRegex() = re and + i = 0 and + re.groupContents(start, end, result.getStart(), result.getEnd()) + } + + override string getPrimaryQLClass() { result = "RegExpGroup" } + } + + // ------------------------------------------------------------------------- + // Zero-width matches and sub-patterns (lookahead/lookbehind) + // ------------------------------------------------------------------------- + + /** + * A zero-width match: an empty group or an assertion. + */ + additional class RegExpZeroWidthMatch extends RegExpGroup { + RegExpZeroWidthMatch() { re.zeroWidthMatch(start, end) } + + override RegExpTerm getChild(int i) { none() } + + override string getPrimaryQLClass() { result = "RegExpZeroWidthMatch" } + } + + /** + * A non-empty zero-width assertion (lookahead or lookbehind). + */ + class RegExpSubPattern extends RegExpZeroWidthMatch { + RegExpSubPattern() { not re.emptyGroup(start, end) } + + /** Gets the operand of this assertion. */ + RegExpTerm getOperand() { + exists(int in_start, int in_end | re.groupContents(start, end, in_start, in_end) | + result.getRegex() = re and + result.getStart() = in_start and + result.getEnd() = in_end + ) + } + } + + /** A lookahead assertion (`(?=...)` or `(?!...)`). */ + abstract class RegExpLookahead extends RegExpSubPattern { } + + /** A positive lookahead assertion (`(?=...)`). */ + class RegExpPositiveLookahead extends RegExpLookahead { + RegExpPositiveLookahead() { re.positiveLookaheadAssertionGroup(start, end, _, _) } + + override string getPrimaryQLClass() { result = "RegExpPositiveLookahead" } + } + + /** A negative lookahead assertion (`(?!...)`). */ + additional class RegExpNegativeLookahead extends RegExpLookahead { + RegExpNegativeLookahead() { re.negativeLookaheadAssertionGroup(start, end, _, _) } + + override string getPrimaryQLClass() { result = "RegExpNegativeLookahead" } + } + + /** A lookbehind assertion (`(?<=...)` or `(?`. + */ + class RegExpBackRef extends RegExpTerm, TRegExpBackRef { + RegExpBackRef() { this = TRegExpBackRef(re, start, end) } + + /** Gets the group number this back-reference refers to, if any. */ + int getNumber() { result = re.getBackrefNumber(start, end) } + + /** Gets the group name this back-reference refers to, if any. */ + string getName() { result = re.getBackrefName(start, end) } + + /** Gets the group that this back-reference refers to. */ + RegExpGroup getGroup() { + this.hasLiteralAndNumber(result.getLiteral(), result.getNumber()) or + this.hasLiteralAndName(result.getLiteral(), result.getName()) + } + + pragma[nomagic] + private predicate hasLiteralAndNumber(RegExpLiteral literal, int number) { + literal = this.getLiteral() and + number = this.getNumber() + } + + pragma[nomagic] + private predicate hasLiteralAndName(RegExpLiteral literal, string name) { + literal = this.getLiteral() and + name = this.getName() + } + + override RegExpTerm getChild(int i) { none() } + + override string getPrimaryQLClass() { result = "RegExpBackRef" } + } + + // ------------------------------------------------------------------------- + // RegExpConstant + // ------------------------------------------------------------------------- + + /** + * A constant regular expression term — a sequence of characters that matches + * a fixed string. Currently this is always a single character (or escape + * sequence). + */ + class RegExpConstant extends RegExpTerm { + string value; + + RegExpConstant() { + this = TRegExpNormalChar(re, start, end) and + not this instanceof RegExpCharacterClassEscape and + not exists(int qstart, int qend | re.qualifiedPart(_, qstart, qend, _, _) | + qstart <= start and end <= qend + ) and + value = this.(RegExpNormalChar).getValue() + } + + /** Holds if this constant is a valid Unicode character (always true). */ + predicate isCharacter() { any() } + + /** Gets the string matched by this constant term. */ + string getValue() { result = value } + + override RegExpTerm getChild(int i) { none() } + + override string getPrimaryQLClass() { result = "RegExpConstant" } + } + + // ------------------------------------------------------------------------- + // Top + // ------------------------------------------------------------------------- + + /** The common supertype of all regex-related elements. */ + class Top = RegExpParent; + + // ------------------------------------------------------------------------- + // Signature predicates + // ------------------------------------------------------------------------- + + /** + * Holds if `term` is an escape class (e.g., `\d`), and `clazz` is the + * character identifying the class (e.g., `"d"`). + */ + predicate isEscapeClass(RegExpTerm term, string clazz) { + exists(RegExpCharacterClassEscape escape | term = escape | escape.getValue() = clazz) + } + + /** + * ECMAScript (`std::regex`) does not support possessive quantifiers, + * so this never holds. + */ + predicate isPossessive(RegExpQuantifier term) { none() } + + /** + * Holds if the regex that `term` is part of is used in a way that ignores + * any leading prefix of the input it is matched against. + * + * Conservative over-approximation: always holds, matching the JavaScript + * implementation's placeholder. TODO (Phase 2): refine using dataflow. + */ + predicate matchesAnyPrefix(RegExpTerm term) { any() } + + /** + * Holds if the regex that `term` is part of is used in a way that ignores + * any trailing suffix of the input it is matched against. + * + * Conservative over-approximation: always holds. TODO (Phase 2): refine. + */ + predicate matchesAnySuffix(RegExpTerm term) { any() } + + /** + * Holds if the regular expression should not be considered by the shared + * analysis. + * + * We exclude: + * - Regexes in files without a relative path (i.e., external/library code). + * - Regexes with an excessive number of `.*` sub-expressions (performance). + */ + predicate isExcluded(RegExpParent parent) { + not exists(parent.getRegex().getFile().getRelativePath()) + or + count(int i | exists(parent.getRegex().getText().regexpFind("\\.\\*", i, _)) | i) > 10 + } + + /** + * Holds if `root` has the `i` flag for case-insensitive matching. + * + * TODO (Phase 2): Implement by detecting `std::regex_constants::icase` at + * the construction site via dataflow. Conservatively returns nothing. + */ + predicate isIgnoreCase(RegExpTerm root) { + root.isRootTerm() and + root.getLiteral().isIgnoreCase() + } + + /** + * Holds if `root` has the `s` (dotAll) flag, making `.` match newlines. + * + * TODO (Phase 2): Implement by detecting the relevant flag at the + * construction site via dataflow. Conservatively returns nothing. + */ + predicate isDotAll(RegExpTerm root) { + root.isRootTerm() and + root.getLiteral().isDotAll() + } +} diff --git a/cpp/ql/lib/semmle/code/cpp/regex/internal/ParseRegExp.qll b/cpp/ql/lib/semmle/code/cpp/regex/internal/ParseRegExp.qll new file mode 100644 index 00000000000..c545d311060 --- /dev/null +++ b/cpp/ql/lib/semmle/code/cpp/regex/internal/ParseRegExp.qll @@ -0,0 +1,840 @@ +/** + * Library for parsing C++ regular expressions. + * + * This parser targets ECMAScript syntax, which is the default mode used by + * `std::regex` (i.e., `std::regex_constants::ECMAScript`). Other syntaxes + * supported by `std::regex` (`basic`, `extended`, `awk`, `grep`, `egrep`) + * as well as third-party libraries (Boost.Regex, PCRE, RE2) differ and are + * not modeled here. + * + * TODO (Phase 2): Restrict `RegExp` to string literals that actually flow + * into a `std::basic_regex` constructor or related function via dataflow, + * rather than treating all string literals as potential regexes. + */ + +import cpp + +/** + * A string literal used (or potentially used) as a regular expression. + * + * For Phase 1 we conservatively treat every `StringLiteral` as a candidate + * regular expression. Phase 2 will narrow this using dataflow to only those + * literals that actually reach a `std::regex` construction or usage site. + */ +class RegExp extends StringLiteral { + RegExp() { + // Restrict to string literals that contain at least one regex metacharacter, + // to keep the working set small and avoid unnecessary computation on plain strings. + // TODO (Phase 2): Replace with a proper dataflow-based check. + exists(this.getValue()) + } + + /** Gets the `i`th character of this regex string. */ + string getChar(int i) { result = this.getValue().charAt(i) } + + /** Gets the text of this regex (the string value of the literal). */ + string getText() { result = this.getValue() } + + // --------------------------------------------------------------------------- + // Escaping + // --------------------------------------------------------------------------- + + /** + * Helper predicate for `escapingChar`. + * Returns `true` if the character at position `pos` is an active backslash + * (i.e., it escapes the next character). Uses a boolean to avoid negation in + * recursive calls. + */ + private boolean escaping(int pos) { + pos = -1 and result = false + or + this.getChar(pos) = "\\" and result = this.escaping(pos - 1).booleanNot() + or + this.getChar(pos) != "\\" and result = false + } + + /** Holds if the character at position `pos` is a backslash that escapes the next character. */ + predicate escapingChar(int pos) { this.escaping(pos) = true } + + // --------------------------------------------------------------------------- + // Character sets (character classes [ ... ] ) + // --------------------------------------------------------------------------- + + /** + * Holds if the (non-escaped) character at position `pos` is the `index`-th + * bracket (`[` or `]`) in the string. + * Result is `true` for `[` and `false` for `]`. + */ + private boolean char_set_delimiter(int index, int pos) { + pos = rank[index](int p | this.nonEscapedCharAt(p) = "[" or this.nonEscapedCharAt(p) = "]") and + ( + this.nonEscapedCharAt(pos) = "[" and result = true + or + this.nonEscapedCharAt(pos) = "]" and result = false + ) + } + + /** + * Helper for `char_set_start/1`. + * Returns `true` if position `pos` is the start of a character class. + */ + boolean char_set_start(int pos) { + exists(int index | + this.char_set_delimiter(index, pos) = true and + ( + index = 1 and result = true + or + index > 1 and + not this.char_set_delimiter(index - 1, _) = false and + result = false + or + exists(int prev_closing_bracket_pos | + this.char_set_delimiter(index - 1, prev_closing_bracket_pos) = false and + if + exists(int pos_before_prev | + if this.getChar(prev_closing_bracket_pos - 1) = "^" + then pos_before_prev = prev_closing_bracket_pos - 2 + else pos_before_prev = prev_closing_bracket_pos - 1 + | + this.char_set_delimiter(index - 2, pos_before_prev) = true + ) + then + exists(int pos_before_prev | + this.char_set_delimiter(index - 2, pos_before_prev) = true + | + result = this.char_set_start(pos_before_prev).booleanNot() + ) + else result = true + ) + ) + ) + } + + /** + * Holds if a character class starts at position `start` with content + * starting at `end` (accounting for optional `^`). + */ + predicate char_set_start(int start, int end) { + this.char_set_start(start) = true and + ( + this.getChar(start + 1) = "^" and end = start + 2 + or + not this.getChar(start + 1) = "^" and end = start + 1 + ) + } + + /** Holds if a character class spans `[start, end)`. */ + predicate charSet(int start, int end) { + exists(int inner_start | + this.char_set_start(start, inner_start) and + not this.char_set_start(_, start) + | + end - 1 = min(int i | this.nonEscapedCharAt(i) = "]" and inner_start < i) + ) + } + + /** An indexed version of `char_set_token`. */ + private predicate char_set_token(int charset_start, int index, int token_start, int token_end) { + token_start = + rank[index](int start, int end | this.char_set_token(charset_start, start, end) | start) and + this.char_set_token(charset_start, token_start, token_end) + } + + /** A single token (character or escape) inside a character class. */ + private predicate char_set_token(int charset_start, int start, int end) { + this.char_set_start(charset_start, start) and + ( + this.escapedCharacter(start, end) + or + exists(this.nonEscapedCharAt(start)) and end = start + 1 + ) + or + this.char_set_token(charset_start, _, start) and + ( + this.escapedCharacter(start, end) + or + exists(this.nonEscapedCharAt(start)) and + end = start + 1 and + not this.getChar(start) = "]" + ) + } + + /** + * Holds if the character class starting at `charset_start` contains a child + * (either a single character or a range) spanning `[start, end)`. + */ + predicate char_set_child(int charset_start, int start, int end) { + this.char_set_token(charset_start, start, end) and + not exists(int range_start, int range_end | + this.charRange(charset_start, range_start, _, _, range_end) and + range_start <= start and + range_end >= end + ) + or + this.charRange(charset_start, start, _, _, end) + } + + /** + * Holds if the character class at `charset_start` contains a character range + * from `[start, lower_end)` to `[upper_start, end)`. + */ + predicate charRange(int charset_start, int start, int lower_end, int upper_start, int end) { + exists(int index | + this.charRangeEnd(charset_start, index) = true and + this.char_set_token(charset_start, index - 2, start, lower_end) and + this.char_set_token(charset_start, index, upper_start, end) + ) + } + + /** + * Helper for `charRange`. Returns `true` if the `index`-th token in the + * character class is the upper bound of a range. + */ + private boolean charRangeEnd(int charset_start, int index) { + this.char_set_token(charset_start, index, _, _) and + ( + index in [1, 2] and result = false + or + index > 2 and + exists(int connector_start | + this.char_set_token(charset_start, index - 1, connector_start, _) and + this.nonEscapedCharAt(connector_start) = "-" and + result = + this.charRangeEnd(charset_start, index - 2) + .booleanNot() + .booleanAnd(this.charRangeEnd(charset_start, index - 1).booleanNot()) + ) + or + not exists(int connector_start | + this.char_set_token(charset_start, index - 1, connector_start, _) and + this.nonEscapedCharAt(connector_start) = "-" + ) and + result = false + ) + } + + // --------------------------------------------------------------------------- + // Characters and escapes + // --------------------------------------------------------------------------- + + /** Gets the non-escaped character at position `i`, if any. */ + string nonEscapedCharAt(int i) { + result = this.getText().charAt(i) and + not exists(int x, int y | this.escapedCharacter(x, y) and i in [x .. y - 1]) + } + + /** Holds if `index` is inside a character class. */ + predicate inCharSet(int index) { + exists(int x, int y | this.charSet(x, y) and index in [x + 1 .. y - 2]) + } + + /** + * Holds if an escaped character sequence spans `[start, end)`. + * This includes hex values, unicode escapes, and simple single-character + * escapes, but excludes back-references. + */ + predicate escapedCharacter(int start, int end) { + this.escapingChar(start) and + not this.numbered_backreference(start, _, _) and + ( + // Hex escape: \xhh + this.getChar(start + 1) = "x" and end = start + 4 + or + // Unicode escape: \uhhhh (ECMAScript) + this.getChar(start + 1) = "u" and + not this.getChar(start + 2) = "{" and + end = start + 6 + or + // Unicode escape with braces: \u{...} (ECMAScript 2015+) + this.getChar(start + 1) = "u" and + this.getChar(start + 2) = "{" and + end - 1 = min(int i | start + 3 <= i and this.getChar(i) = "}") + or + // Octal (legacy): \0 (NUL), \ooo + this.getChar(start + 1) = "0" and + not this.isDecimalDigit(start + 2) and + end = start + 2 + or + // Simple escape: \n, \r, \t, \f, \v, \b (inside char class), \\, \., etc. + not this.getChar(start + 1) in ["x", "u", "0"] and + not this.isDecimalDigit(start + 1) and + not this.getChar(start + 1) = "k" and + // \k is a named backreference, handled separately + end = start + 2 + ) + } + + pragma[inline] + private predicate isDecimalDigit(int index) { + this.getChar(index) = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"] + } + + /** + * A 'simple' character is one that does not affect parsing of the regex + * (i.e., it is not a metacharacter). + */ + private predicate simpleCharacter(int start, int end) { + end = start + 1 and + not this.charSet(start, _) and + not this.charSet(_, start + 1) and + exists(string c | c = this.getChar(start) | + exists(int x, int y, int z | + this.charSet(x, z) and + this.char_set_start(x, y) + | + start = y + or + start = z - 2 + or + start > y and start < z - 2 and not this.charRange(_, _, start, end, _) + ) + or + not this.inCharSet(start) and + not c = "(" and + not c = "[" and + not c = ")" and + not c = "|" and + not this.qualifier(start, _, _, _) + ) + } + + /** Holds if a simple or escaped character spans `[start, end)`. */ + predicate character(int start, int end) { + ( + this.simpleCharacter(start, end) and + not exists(int x, int y | this.escapedCharacter(x, y) and x <= start and y >= end) + or + this.escapedCharacter(start, end) + ) and + not exists(int x, int y | this.group_start(x, y) and x <= start and y >= end) and + not exists(int x, int y | this.backreference(x, y) and x <= start and y >= end) + } + + /** Holds if a normal (non-special) character spans `[start, end)`. */ + predicate normalCharacter(int start, int end) { + end = start + 1 and + this.character(start, end) and + not this.specialCharacter(start, end, _) + } + + /** + * Holds if the character at `[start, end)` is a special (metacharacter) in + * ECMAScript regex syntax. `char` is the canonical representation. + * + * Special characters are: `.`, `^`, `$`, `\b`, `\B`. + * (Note: `\A`, `\Z`, `\G` are Ruby/Python-specific and not part of ECMAScript.) + */ + predicate specialCharacter(int start, int end, string char) { + not this.inCharSet(start) and + this.character(start, end) and + ( + end = start + 1 and + char = this.getChar(start) and + (char = "$" or char = "^" or char = ".") + or + end = start + 2 and + this.escapingChar(start) and + char = this.getText().substring(start, end) and + (char = "\\b" or char = "\\B") + ) + } + + /** Holds if `[start, end)` is a maximal run of normal characters (a "constant"). */ + predicate normalCharacterSequence(int start, int end) { + // A single normal character inside a character class stands alone + this.normalCharacter(start, end) and + this.inCharSet(start) + or + // A maximal run of normal characters outside a character class + exists(int s, int e | + e = max(int i | this.normalCharacterRun(s, i)) and + not this.inCharSet(s) + | + if this.qualifier(e, _, _, _) + then + end = e and start = e - 1 + or + end = e - 1 and start = s and start < end + else ( + end = e and + start = s + ) + ) + } + + private predicate normalCharacterRun(int start, int end) { + ( + this.normalCharacterRun(start, end - 1) + or + start = end - 1 and not this.normalCharacter(start - 1, start) + ) and + this.normalCharacter(end - 1, end) + } + + private predicate characterItem(int start, int end) { + this.normalCharacterSequence(start, end) or + this.escapedCharacter(start, end) or + this.specialCharacter(start, end, _) + } + + // --------------------------------------------------------------------------- + // Quantifiers + // --------------------------------------------------------------------------- + + /** + * Holds if a repetition quantifier spans `[start, end)`, with `maybe_empty` + * indicating whether zero repetitions are possible, and `may_repeat_forever` + * indicating whether the count is unbounded. + * + * In ECMAScript, lazy quantifiers (`*?`, `+?`, `??`, `{n,m}?`) are treated + * as having the same `maybe_empty`/`may_repeat_forever` as their greedy + * counterparts from the perspective of ReDoS analysis. + */ + predicate qualifier(int start, int end, boolean maybe_empty, boolean may_repeat_forever) { + this.short_qualifier(start, end, maybe_empty, may_repeat_forever) and + not this.getChar(end) = "?" + or + exists(int short_end | this.short_qualifier(start, short_end, maybe_empty, may_repeat_forever) | + if this.getChar(short_end) = "?" then end = short_end + 1 else end = short_end + ) + } + + /** + * Holds if `[start, end)` is a greedy (non-lazy) quantifier. The `maybe_empty` + * and `may_repeat_forever` booleans characterize the quantifier type. + */ + predicate short_qualifier(int start, int end, boolean maybe_empty, boolean may_repeat_forever) { + ( + this.getChar(start) = "+" and maybe_empty = false and may_repeat_forever = true + or + this.getChar(start) = "*" and maybe_empty = true and may_repeat_forever = true + or + this.getChar(start) = "?" and maybe_empty = true and may_repeat_forever = false + ) and + end = start + 1 + or + exists(string lower, string upper | + this.multiples(start, end, lower, upper) and + (if lower = "" or lower.toInt() = 0 then maybe_empty = true else maybe_empty = false) and + if upper = "" then may_repeat_forever = true else may_repeat_forever = false + ) + } + + /** + * Holds if `[start, end)` is a `{n}`, `{n,m}`, or `{n,}` quantifier. + * `lower` and `upper` are the textual lower and upper bounds; an empty + * `upper` means "no upper bound". + */ + predicate multiples(int start, int end, string lower, string upper) { + exists(string text, string match, string inner | + text = this.getText() and + end = start + match.length() and + inner = match.substring(1, match.length() - 1) + | + match = text.regexpFind("\\{[0-9]+\\}", _, start) and + lower = inner and + upper = lower + or + match = text.regexpFind("\\{[0-9]*,[0-9]*\\}", _, start) and + exists(int commaIndex | + commaIndex = inner.indexOf(",") and + lower = inner.prefix(commaIndex) and + upper = inner.suffix(commaIndex + 1) + ) + ) + } + + /** + * Holds if `[start, end)` is a qualified item (base item + quantifier). + */ + predicate qualifiedItem(int start, int end, boolean maybe_empty, boolean may_repeat_forever) { + this.qualifiedPart(start, _, end, maybe_empty, may_repeat_forever) + } + + /** + * Holds if the base item spans `[start, part_end)` and the qualifier spans + * `[part_end, end)`. + */ + predicate qualifiedPart( + int start, int part_end, int end, boolean maybe_empty, boolean may_repeat_forever + ) { + this.baseItem(start, part_end) and + this.qualifier(part_end, end, maybe_empty, may_repeat_forever) + } + + /** Holds if `[start, end)` is a single regex item (possibly qualified). */ + predicate item(int start, int end) { + this.qualifiedItem(start, end, _, _) + or + this.baseItem(start, end) and not this.qualifier(end, _, _, _) + } + + // --------------------------------------------------------------------------- + // Groups + // --------------------------------------------------------------------------- + + private predicate isOptionDivider(int i) { this.nonEscapedCharAt(i) = "|" } + + private predicate isGroupEnd(int i) { this.nonEscapedCharAt(i) = ")" and not this.inCharSet(i) } + + private predicate isGroupStart(int i) { this.nonEscapedCharAt(i) = "(" and not this.inCharSet(i) } + + /** + * Matches the start of any group construct, yielding `[start, end)` where + * `end` is the first position of the group content. + */ + private predicate group_start(int start, int end) { + this.non_capturing_group_start(start, end) + or + this.ecma_named_group_start(start, end) + or + this.lookahead_assertion_start(start, end) + or + this.negative_lookahead_assertion_start(start, end) + or + this.lookbehind_assertion_start(start, end) + or + this.negative_lookbehind_assertion_start(start, end) + or + this.simple_group_start(start, end) + } + + /** `(?:...)` – non-capturing group. */ + private predicate non_capturing_group_start(int start, int end) { + this.isGroupStart(start) and + this.getChar(start + 1) = "?" and + this.getChar(start + 2) = ":" and + end = start + 3 + } + + /** `(...)` – simple capturing group. */ + private predicate simple_group_start(int start, int end) { + this.isGroupStart(start) and + this.getChar(start + 1) != "?" and + end = start + 1 + } + + /** + * `(?...)` – ECMAScript named capturing group. + * The group name spans from `start+3` to the `>` character. + */ + private predicate ecma_named_group_start(int start, int end) { + this.isGroupStart(start) and + this.getChar(start + 1) = "?" and + this.getChar(start + 2) = "<" and + not this.getChar(start + 3) = "=" and + not this.getChar(start + 3) = "!" and + exists(int name_end | + name_end = min(int i | i > start + 3 and this.getChar(i) = ">") and + end = name_end + 1 + ) + } + + /** `(?=...)` – positive lookahead. */ + predicate lookahead_assertion_start(int start, int end) { + this.isGroupStart(start) and + this.getChar(start + 1) = "?" and + this.getChar(start + 2) = "=" and + end = start + 3 + } + + /** `(?!...)` – negative lookahead. */ + predicate negative_lookahead_assertion_start(int start, int end) { + this.isGroupStart(start) and + this.getChar(start + 1) = "?" and + this.getChar(start + 2) = "!" and + end = start + 3 + } + + /** `(?<=...)` – positive lookbehind. */ + predicate lookbehind_assertion_start(int start, int end) { + this.isGroupStart(start) and + this.getChar(start + 1) = "?" and + this.getChar(start + 2) = "<" and + this.getChar(start + 3) = "=" and + end = start + 4 + } + + /** `(?...)` still get a + * number in ECMAScript, but `(?:...)` does not. + */ + int getGroupNumber(int start, int end) { + this.group(start, end) and + not this.non_capturing_group_start(start, _) and + not this.lookahead_assertion_start(start, _) and + not this.negative_lookahead_assertion_start(start, _) and + not this.lookbehind_assertion_start(start, _) and + not this.negative_lookbehind_assertion_start(start, _) and + result = + count(int i | + this.group(i, _) and + i < start and + not this.non_capturing_group_start(i, _) and + not this.lookahead_assertion_start(i, _) and + not this.negative_lookahead_assertion_start(i, _) and + not this.lookbehind_assertion_start(i, _) and + not this.negative_lookbehind_assertion_start(i, _) + ) + 1 + } + + /** + * Gets the name of the ECMAScript named group at `[start, end)`, if any. + * Named groups have the form `(?...)`. + */ + string getGroupName(int start, int end) { + this.group(start, end) and + exists(int name_end | + this.ecma_named_group_start(start, name_end) and + result = this.getText().substring(start + 3, name_end - 1) + ) + } + + /** Holds if a zero-width match group is at `[start, end)`. */ + predicate zeroWidthMatch(int start, int end) { + this.emptyGroup(start, end) + or + this.negativeLookaheadAssertionGroup(start, end, _, _) + or + this.positiveLookaheadAssertionGroup(start, end, _, _) + or + this.positiveLookbehindAssertionGroup(start, end, _, _) + or + this.negativeLookbehindAssertionGroup(start, end, _, _) + } + + /** Holds if a positive lookahead `(?=...)` is at `[start, end)`. */ + predicate positiveLookaheadAssertionGroup(int start, int end, int in_start, int in_end) { + this.lookahead_assertion_start(start, in_start) and + this.groupContents(start, end, in_start, in_end) + } + + /** Holds if a negative lookahead `(?!...)` is at `[start, end)`. */ + predicate negativeLookaheadAssertionGroup(int start, int end, int in_start, int in_end) { + this.negative_lookahead_assertion_start(start, in_start) and + this.groupContents(start, end, in_start, in_end) + } + + /** Holds if a positive lookbehind `(?<=...)` is at `[start, end)`. */ + predicate positiveLookbehindAssertionGroup(int start, int end, int in_start, int in_end) { + this.lookbehind_assertion_start(start, in_start) and + this.groupContents(start, end, in_start, in_end) + } + + /** Holds if a negative lookbehind `(?` spans `[start, end)` + * with the given `name`. + */ + private predicate named_backreference(int start, int end, string name) { + this.escapingChar(start) and + this.getChar(start + 1) = "k" and + this.getChar(start + 2) = "<" and + exists(int name_end | + name_end = min(int i | i > start + 3 and this.getChar(i) = ">") and + end = name_end + 1 and + name = this.getText().substring(start + 3, name_end) + ) + } + + /** Holds if a back-reference spans `[start, end)`. */ + predicate backreference(int start, int end) { + this.numbered_backreference(start, end, _) + or + this.named_backreference(start, end, _) + } + + /** Gets the number of the back-reference at `[start, end)`, if any. */ + int getBackrefNumber(int start, int end) { this.numbered_backreference(start, end, result) } + + /** Gets the name of the back-reference at `[start, end)`, if any. */ + string getBackrefName(int start, int end) { this.named_backreference(start, end, result) } + + // --------------------------------------------------------------------------- + // Sequences and alternations + // --------------------------------------------------------------------------- + + private predicate baseItem(int start, int end) { + this.characterItem(start, end) and + not exists(int x, int y | this.charSet(x, y) and x <= start and y >= end) + or + this.group(start, end) + or + this.charSet(start, end) + or + this.backreference(start, end) + } + + private predicate subsequence(int start, int end) { + ( + start = 0 or + this.group_start(_, start) or + this.isOptionDivider(start - 1) + ) and + this.item(start, end) + or + exists(int mid | + this.subsequence(start, mid) and + this.item(mid, end) + ) + } + + /** + * Holds if `[start, end)` is a sequence of two or more items. + */ + predicate sequence(int start, int end) { + this.sequenceOrQualified(start, end) and + not this.qualifiedItem(start, end, _, _) + } + + private predicate sequenceOrQualified(int start, int end) { + this.subsequence(start, end) and + not this.item_start(end) + } + + private predicate item_start(int start) { + this.characterItem(start, _) or + this.isGroupStart(start) or + this.charSet(start, _) or + this.backreference(start, _) + } + + private predicate item_end(int end) { + this.characterItem(_, end) + or + exists(int endm1 | this.isGroupEnd(endm1) and end = endm1 + 1) + or + this.charSet(_, end) + or + this.qualifier(_, end, _, _) + } + + private predicate top_level(int start, int end) { + this.subalternation(start, end, _) and + not this.isOptionDivider(end) + } + + private predicate subalternation(int start, int end, int item_start) { + this.sequenceOrQualified(start, end) and + not this.isOptionDivider(start - 1) and + item_start = start + or + start = end and + not this.item_end(start) and + this.isOptionDivider(end) and + item_start = start + or + exists(int mid | + this.subalternation(start, mid, _) and + this.isOptionDivider(mid) and + item_start = mid + 1 + | + this.sequenceOrQualified(item_start, end) + or + not this.item_start(end) and end = item_start + ) + } + + /** Holds if `[start, end)` is an alternation (two or more options separated by `|`). */ + predicate alternation(int start, int end) { + this.top_level(start, end) and + exists(int less | this.subalternation(start, less, _) and less < end) + } + + /** + * Holds if `[start, end)` is an alternation, and `[part_start, part_end)` is + * one of its options. + */ + predicate alternationOption(int start, int end, int part_start, int part_end) { + this.alternation(start, end) and + this.subalternation(start, part_end, part_start) + } + + // --------------------------------------------------------------------------- + // Failure to parse + // --------------------------------------------------------------------------- + + /** + * Holds if the character at position `i` could not be parsed as part of any + * top-level regex construct. + */ + predicate failedToParse(int i) { + exists(this.getChar(i)) and + not exists(int start, int end | + this.top_level(start, end) and + start <= i and + end > i + ) + } +} diff --git a/cpp/ql/test/library-tests/regex/CompileCheck.expected b/cpp/ql/test/library-tests/regex/CompileCheck.expected new file mode 100644 index 00000000000..e69de29bb2d diff --git a/cpp/ql/test/library-tests/regex/CompileCheck.ql b/cpp/ql/test/library-tests/regex/CompileCheck.ql new file mode 100644 index 00000000000..35a4cc4dfad --- /dev/null +++ b/cpp/ql/test/library-tests/regex/CompileCheck.ql @@ -0,0 +1,24 @@ +/** + * Compile-check: verifies that the C++ `RegexTreeView` implementation satisfies + * the `RegexTreeViewSig` signature and can be used to instantiate the shared + * ReDoS analysis engines. + * + * This query produces no results; its sole purpose is to ensure the modules + * compile without type errors. + * + * @kind table + */ + +import cpp +import semmle.code.cpp.regex.RegexTreeView +private import codeql.regex.nfa.SuperlinearBackTracking as SuperlinearBackTracking +private import codeql.regex.nfa.NfaUtils as NfaUtils + +// Instantiate the shared analysis modules with our RegexTreeView. +// If RegexTreeView does not satisfy RegexTreeViewSig, these lines cause a compile error. +private module TestSuperlinear = SuperlinearBackTracking::Make; + +private module TestNfaUtils = NfaUtils::Make; + +// No results are selected; this is a compile-only check. +select "x" where 1 = 0 diff --git a/cpp/ql/test/library-tests/regex/Consistency.expected b/cpp/ql/test/library-tests/regex/Consistency.expected new file mode 100644 index 00000000000..e69de29bb2d diff --git a/cpp/ql/test/library-tests/regex/Consistency.ql b/cpp/ql/test/library-tests/regex/Consistency.ql new file mode 100644 index 00000000000..1380871c313 --- /dev/null +++ b/cpp/ql/test/library-tests/regex/Consistency.ql @@ -0,0 +1,13 @@ +/** + * Flags positions in regular expressions that failed to parse. + * An empty expected output confirms that all regex constructs in the test + * file were parsed successfully. + */ + +import cpp +import semmle.code.cpp.regex.RegexTreeView::RegexTreeView as RTV +import semmle.code.cpp.regex.internal.ParseRegExp + +from RegExp re, int i +where re.failedToParse(i) +select re, i, "Failed to parse character at offset " + i + " in " + re.getValue() diff --git a/cpp/ql/test/library-tests/regex/parse.expected b/cpp/ql/test/library-tests/regex/parse.expected new file mode 100644 index 00000000000..e69de29bb2d diff --git a/cpp/ql/test/library-tests/regex/parse.ql b/cpp/ql/test/library-tests/regex/parse.ql new file mode 100644 index 00000000000..e65e0cd324a --- /dev/null +++ b/cpp/ql/test/library-tests/regex/parse.ql @@ -0,0 +1,27 @@ +/** + * @kind graph + */ + +import cpp +import semmle.code.cpp.regex.RegexTreeView + +query predicate nodes(RegExpTerm n, string attr, string val) { + attr = "semmle.label" and + val = "[" + concat(n.getAPrimaryQlClass(), ", ") + "] " + n.toString() + or + attr = "semmle.order" and + val = + any(int i | + n = + rank[i](RegExpTerm t, string fp, int sl, int sc, int el, int ec | + t.hasLocationInfo(fp, sl, sc, el, ec) + | + t order by fp, sl, sc, el, ec, t.toString() + ) + ).toString() +} + +query predicate edges(RegExpTerm pred, RegExpTerm succ, string attr, string val) { + attr in ["semmle.label", "semmle.order"] and + val = any(int i | succ = pred.getChild(i)).toString() +} diff --git a/cpp/ql/test/library-tests/regex/regexp.expected b/cpp/ql/test/library-tests/regex/regexp.expected new file mode 100644 index 00000000000..e69de29bb2d diff --git a/cpp/ql/test/library-tests/regex/regexp.ql b/cpp/ql/test/library-tests/regex/regexp.ql new file mode 100644 index 00000000000..bd042354d9f --- /dev/null +++ b/cpp/ql/test/library-tests/regex/regexp.ql @@ -0,0 +1,10 @@ +import cpp +import semmle.code.cpp.regex.RegexTreeView + +query predicate groupName(RegExpGroup g, string name) { name = g.getName() } + +query predicate groupNumber(RegExpGroup g, int number) { number = g.getNumber() } + +query predicate term(RegExpTerm t, string c) { c = t.getPrimaryQlClasses() } + +query predicate regExpNormalCharValue(RegExpNormalChar n, string value) { value = n.getValue() } diff --git a/cpp/ql/test/library-tests/regex/test.cpp b/cpp/ql/test/library-tests/regex/test.cpp new file mode 100644 index 00000000000..4e24751b23d --- /dev/null +++ b/cpp/ql/test/library-tests/regex/test.cpp @@ -0,0 +1,83 @@ +#include +#include + +// Basic sequence +void basic_sequence() { + std::regex r1("abc"); +} + +// Repetition +void repetition() { + std::regex r1("a*b+c?d"); + std::regex r2("a{4,8}"); + std::regex r3("a{3,}"); + std::regex r4("a{7}"); +} + +// Alternation +void alternation() { + std::regex r1("foo|bar"); +} + +// Character classes +void character_classes() { + std::regex r1("[abc]"); + std::regex r2("[a-fA-F0-9_]"); + std::regex r3("[\\w]+"); + std::regex r4("[^A-Z]"); +} + +// Meta-character classes and dot +void meta_classes() { + std::regex r1(".*"); + std::regex r2("\\w+\\W"); + std::regex r3("\\s\\S"); + std::regex r4("\\d\\D"); +} + +// Anchors +void anchors() { + std::regex r1("^abc$"); + std::regex r2("\\babc\\B"); +} + +// Groups +void groups() { + std::regex r1("(foo)*bar"); + std::regex r2("fo(o|b)ar"); + std::regex r3("(a|b|cd)e"); + std::regex r4("(?::+)\\w"); // Non-capturing group +} + +// Named groups (ECMAScript style: (?...)) +void named_groups() { + std::regex r1("(?\\w+)"); + std::regex r2("(?[a-z]+)(?[0-9]+)"); +} + +// Backreferences +void backreferences() { + std::regex r1("(a+)b+\\1"); + std::regex r2("(?q+)\\s+\\k+"); +} + +// Lookahead and lookbehind +void lookahead_lookbehind() { + std::regex r1("(?=\\w)abc"); + std::regex r2("(?!\\d)abc"); + std::regex r3("abc(?<=\\w)"); + std::regex r4("abc(?