From cebddbdd42c4e14bfddd33330f80653c44781d3f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 21 Jul 2026 10:22:50 +0000 Subject: [PATCH] Commit 7: Implement POSIX collating symbol and equivalence class support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add posixCollatingSymbol([.x.]) and posixEquivalenceClass([=x=]) predicates to ParseRegExp.qll, enabling correct tokenization of all three POSIX bracket atom types inside character classes: - [:name:] — already handled, unchanged - [.x.] — new: posixCollatingSymbol predicate - [=x=] — new: posixEquivalenceClass predicate Refactored: - charSetDelimiter: use inAnyPosixBracket helper (covers all three types) - charSet closing: use inAnyPosixBracket helper - inPosixBracket: updated to cover all three types - simpleCharacter: updated to exclude all three types - namedCharacterProperty: includes posixCollatingSymbol and posixEquivalenceClass - inAnyPosixBracket: new private helper; uses pos in [x..y-1] for QL binding Before this commit (commit 6): [[.a.]] → RegExpCharacterClass([[.a.]) + stray ] (WRONG) [[=a=]] → RegExpCharacterClass([[=a=]) + stray ] (WRONG) After this commit: [[.a.]] → RegExpCharacterClass containing RegExpNamedCharacterProperty [.a.] [[=a=]] → RegExpCharacterClass containing RegExpNamedCharacterProperty [=a=] Regenerated parse.expected and regexp.expected via codeql test run --learn (CodeQL 2.26.1); all 2 tests pass. --- .../code/cpp/regex/internal/ParseRegExp.qll | 81 +++++++++++++++---- .../test/library-tests/regex/parse.expected | 42 ++-------- .../test/library-tests/regex/regexp.expected | 28 +------ 3 files changed, 76 insertions(+), 75 deletions(-) diff --git a/cpp/ql/lib/semmle/code/cpp/regex/internal/ParseRegExp.qll b/cpp/ql/lib/semmle/code/cpp/regex/internal/ParseRegExp.qll index aa75616b52b..ad140bd9489 100644 --- a/cpp/ql/lib/semmle/code/cpp/regex/internal/ParseRegExp.qll +++ b/cpp/ql/lib/semmle/code/cpp/regex/internal/ParseRegExp.qll @@ -102,11 +102,9 @@ class RegExp extends StringLiteral { pos = rank[index](int p | (this.nonEscapedCharAt(p) = "[" or this.nonEscapedCharAt(p) = "]") and - // Brackets that art part of POSIX expressions should not count as + // Brackets that are part of POSIX expressions should not count as // char-set delimiters. - not exists(int x, int y | - this.posixStyleNamedCharacterProperty(x, y, _) and pos >= x and pos < y - ) + not this.inAnyPosixBracket(p) ) and ( this.nonEscapedCharAt(pos) = "[" and result = true @@ -136,9 +134,7 @@ class RegExp extends StringLiteral { min(int e | e > innerStart and this.nonEscapedCharAt(e) = "]" and - not exists(int x, int y | - this.posixStyleNamedCharacterProperty(x, y, _) and e >= x and e < y - ) + not this.inAnyPosixBracket(e) | e ) @@ -289,7 +285,9 @@ class RegExp extends StringLiteral { /** Matches named character properties such as `\p{Word}` and `[[:digit:]]` */ predicate namedCharacterProperty(int start, int end, string name) { this.pStyleNamedCharacterProperty(start, end, name) or - this.posixStyleNamedCharacterProperty(start, end, name) + this.posixStyleNamedCharacterProperty(start, end, name) or + this.posixCollatingSymbol(start, end, name) or + this.posixEquivalenceClass(start, end, name) } /** Gets the name of the character property in start,end */ @@ -297,6 +295,21 @@ class RegExp extends StringLiteral { this.namedCharacterProperty(start, end, result) } + /** + * Holds if the position `pos` falls inside any POSIX bracket atom + * (`[:name:]`, `[.x.]`, or `[=x=]`). + * Used to prevent the inner brackets from being treated as charSet delimiters. + */ + private predicate inAnyPosixBracket(int pos) { + exists(int x, int y | + this.posixStyleNamedCharacterProperty(x, y, _) or + this.posixCollatingSymbol(x, y, _) or + this.posixEquivalenceClass(x, y, _) + | + pos in [x .. y - 1] + ) + } + /** Matches a POSIX bracket expression such as `[:alnum:]` within a character class. */ private predicate posixStyleNamedCharacterProperty(int start, int end, string name) { this.getChar(start) = "[" and @@ -318,6 +331,42 @@ class RegExp extends StringLiteral { ) } + /** + * Matches a POSIX collating symbol such as `[.a.]` within a character class. + * Example: `[[.a.]]` — the `[.a.]` atom nested inside the outer bracket. + */ + private predicate posixCollatingSymbol(int start, int end, string name) { + this.getChar(start) = "[" and + this.getChar(start + 1) = "." and + end = + min(int e | + e > start and + this.getChar(e - 2) = "." and + this.getChar(e - 1) = "]" + | + e + ) and + name = this.getText().substring(start + 2, end - 2) + } + + /** + * Matches a POSIX equivalence class such as `[=a=]` within a character class. + * Example: `[[=a=]]` — the `[=a=]` atom nested inside the outer bracket. + */ + private predicate posixEquivalenceClass(int start, int end, string name) { + this.getChar(start) = "[" and + this.getChar(start + 1) = "=" and + end = + min(int e | + e > start and + this.getChar(e - 2) = "=" and + this.getChar(e - 1) = "]" + | + e + ) and + name = this.getText().substring(start + 2, end - 2) + } + /** * Matches named character properties. For example: * - `\p{Space}` @@ -398,11 +447,17 @@ class RegExp extends StringLiteral { } /** - * Holds if the character at `index` is inside a posix bracket. + * Holds if the character at `index` is inside a posix bracket + * (`[:name:]`, `[.x.]`, or `[=x=]`). */ predicate inPosixBracket(int index) { exists(int x, int y | - this.posixStyleNamedCharacterProperty(x, y, _) and index in [x + 1 .. y - 2] + ( + this.posixStyleNamedCharacterProperty(x, y, _) or + this.posixCollatingSymbol(x, y, _) or + this.posixEquivalenceClass(x, y, _) + ) and + index in [x + 1 .. y - 2] ) } @@ -413,11 +468,7 @@ class RegExp extends StringLiteral { end = start + 1 and not this.charSet(start, _) and not this.charSet(_, start + 1) and - not exists(int x, int y | - this.posixStyleNamedCharacterProperty(x, y, _) and - start >= x and - end <= y - ) and + not this.inAnyPosixBracket(start) and exists(string c | c = this.getChar(start) | exists(int x, int y, int z | this.charSet(x, z) and diff --git a/cpp/ql/test/library-tests/regex/parse.expected b/cpp/ql/test/library-tests/regex/parse.expected index 96c377528c3..02101deb0fb 100644 --- a/cpp/ql/test/library-tests/regex/parse.expected +++ b/cpp/ql/test/library-tests/regex/parse.expected @@ -245,45 +245,15 @@ test.cpp: # 76| [RegExpNamedCharacterProperty] [:space:] -# 79| [RegExpCharacterClass] [[.a.] -#-----| 0 -> [RegExpConstant, RegExpNormalChar] [ -#-----| 1 -> [RegExpConstant, RegExpNormalChar] . -#-----| 2 -> [RegExpConstant, RegExpNormalChar] a -#-----| 3 -> [RegExpConstant, RegExpNormalChar] . +# 79| [RegExpCharacterClass] [[.a.]] +#-----| 0 -> [RegExpNamedCharacterProperty] [.a.] -# 79| [RegExpSequence] [[.a.]] -#-----| 0 -> [RegExpCharacterClass] [[.a.] -#-----| 1 -> [RegExpConstant, RegExpNormalChar] ] +# 79| [RegExpNamedCharacterProperty] [.a.] -# 79| [RegExpConstant, RegExpNormalChar] [ +# 82| [RegExpCharacterClass] [[=a=]] +#-----| 0 -> [RegExpNamedCharacterProperty] [=a=] -# 79| [RegExpConstant, RegExpNormalChar] . - -# 79| [RegExpConstant, RegExpNormalChar] a - -# 79| [RegExpConstant, RegExpNormalChar] . - -# 79| [RegExpConstant, RegExpNormalChar] ] - -# 82| [RegExpCharacterClass] [[=a=] -#-----| 0 -> [RegExpConstant, RegExpNormalChar] [ -#-----| 1 -> [RegExpConstant, RegExpNormalChar] = -#-----| 2 -> [RegExpConstant, RegExpNormalChar] a -#-----| 3 -> [RegExpConstant, RegExpNormalChar] = - -# 82| [RegExpSequence] [[=a=]] -#-----| 0 -> [RegExpCharacterClass] [[=a=] -#-----| 1 -> [RegExpConstant, RegExpNormalChar] ] - -# 82| [RegExpConstant, RegExpNormalChar] [ - -# 82| [RegExpConstant, RegExpNormalChar] = - -# 82| [RegExpConstant, RegExpNormalChar] a - -# 82| [RegExpConstant, RegExpNormalChar] = - -# 82| [RegExpConstant, RegExpNormalChar] ] +# 82| [RegExpNamedCharacterProperty] [=a=] # 85| [RegExpCharacterClass] [[:alpha:]0-9] #-----| 0 -> [RegExpNamedCharacterProperty] [:alpha:] diff --git a/cpp/ql/test/library-tests/regex/regexp.expected b/cpp/ql/test/library-tests/regex/regexp.expected index a1d8fc3ec8b..092c5fbb446 100644 --- a/cpp/ql/test/library-tests/regex/regexp.expected +++ b/cpp/ql/test/library-tests/regex/regexp.expected @@ -100,20 +100,10 @@ term | test.cpp:76:29:76:40 | [a[:space:]] | RegExpCharacterClass | | test.cpp:76:30:76:30 | a | RegExpConstant,RegExpNormalChar | | test.cpp:76:31:76:39 | [:space:] | RegExpNamedCharacterProperty | -| test.cpp:79:28:79:33 | [[.a.] | RegExpCharacterClass | -| test.cpp:79:28:79:34 | [[.a.]] | RegExpSequence | -| test.cpp:79:29:79:29 | [ | RegExpConstant,RegExpNormalChar | -| test.cpp:79:30:79:30 | . | RegExpConstant,RegExpNormalChar | -| test.cpp:79:31:79:31 | a | RegExpConstant,RegExpNormalChar | -| test.cpp:79:32:79:32 | . | RegExpConstant,RegExpNormalChar | -| test.cpp:79:34:79:34 | ] | RegExpConstant,RegExpNormalChar | -| test.cpp:82:29:82:34 | [[=a=] | RegExpCharacterClass | -| test.cpp:82:29:82:35 | [[=a=]] | RegExpSequence | -| test.cpp:82:30:82:30 | [ | RegExpConstant,RegExpNormalChar | -| test.cpp:82:31:82:31 | = | RegExpConstant,RegExpNormalChar | -| test.cpp:82:32:82:32 | a | RegExpConstant,RegExpNormalChar | -| test.cpp:82:33:82:33 | = | RegExpConstant,RegExpNormalChar | -| test.cpp:82:35:82:35 | ] | RegExpConstant,RegExpNormalChar | +| test.cpp:79:28:79:34 | [[.a.]] | RegExpCharacterClass | +| test.cpp:79:29:79:33 | [.a.] | RegExpNamedCharacterProperty | +| test.cpp:82:29:82:35 | [[=a=]] | RegExpCharacterClass | +| test.cpp:82:30:82:34 | [=a=] | RegExpNamedCharacterProperty | | test.cpp:85:29:85:42 | [[:alpha:]0-9] | RegExpCharacterClass | | test.cpp:85:30:85:38 | [:alpha:] | RegExpNamedCharacterProperty | | test.cpp:85:39:85:39 | 0 | RegExpConstant,RegExpNormalChar | @@ -252,16 +242,6 @@ regExpNormalCharValue | test.cpp:57:28:57:28 | f | f | | test.cpp:57:30:57:33 | A-F] | A-F] | | test.cpp:76:30:76:30 | a | a | -| test.cpp:79:29:79:29 | [ | [ | -| test.cpp:79:30:79:30 | . | . | -| test.cpp:79:31:79:31 | a | a | -| test.cpp:79:32:79:32 | . | . | -| test.cpp:79:34:79:34 | ] | ] | -| test.cpp:82:30:82:30 | [ | [ | -| test.cpp:82:31:82:31 | = | = | -| test.cpp:82:32:82:32 | a | a | -| test.cpp:82:33:82:33 | = | = | -| test.cpp:82:35:82:35 | ] | ] | | test.cpp:85:39:85:39 | 0 | 0 | | test.cpp:85:41:85:41 | 9 | 9 | | test.cpp:89:23:89:24 | \\w | w |