Ruby/Python: regex parser: group sequences of 'normal' characters

2026-05-03 04:39:29 +02:00 · 2022-02-22 10:51:47 +01:00
parent 36e02ae9ac
commit 69ed121ecb
9 changed files with 166 additions and 231 deletions
--- a/ruby/ql/test/library-tests/ast/Ast.expected
+++ b/ruby/ql/test/library-tests/ast/Ast.expected
@@ -973,10 +973,8 @@ control/cases.rb:
 #   92|         getParsed: [RegExpSequence] .*abc[0-9]
 #   92|           0: [RegExpStar] .*
 #   92|             0: [RegExpDot] .
-#   92|           1: [RegExpConstant, RegExpNormalChar] a
-#   92|           2: [RegExpConstant, RegExpNormalChar] b
-#   92|           3: [RegExpConstant, RegExpNormalChar] c
-#   92|           4: [RegExpCharacterClass] [0-9]
+#   92|           1: [RegExpConstant, RegExpNormalChar] abc
+#   92|           2: [RegExpCharacterClass] [0-9]
 #   92|             0: [RegExpCharacterRange] 0-9
 #   92|               0: [RegExpConstant, RegExpNormalChar] 0
 #   92|               1: [RegExpConstant, RegExpNormalChar] 9
@@ -1823,47 +1821,25 @@ literals/literals.rb:
 #  133|         getAnOperand/getArgument/getRightOperand: [IntegerLiteral] 4
 #  136|   getStmt: [RegExpLiteral] //
 #  137|   getStmt: [RegExpLiteral] /foo/
-#  137|     getParsed: [RegExpSequence] foo
-#  137|       0: [RegExpConstant, RegExpNormalChar] f
-#  137|       1: [RegExpConstant, RegExpNormalChar] o
-#  137|       2: [RegExpConstant, RegExpNormalChar] o
+#  137|     getParsed: [RegExpConstant, RegExpNormalChar] foo
 #  137|     getComponent: [RegExpTextComponent] foo
 #  138|   getStmt: [RegExpLiteral] /foo/
-#  138|     getParsed: [RegExpSequence] foo
-#  138|       0: [RegExpConstant, RegExpNormalChar] f
-#  138|       1: [RegExpConstant, RegExpNormalChar] o
-#  138|       2: [RegExpConstant, RegExpNormalChar] o
+#  138|     getParsed: [RegExpConstant, RegExpNormalChar] foo
 #  138|     getComponent: [RegExpTextComponent] foo
 #  139|   getStmt: [RegExpLiteral] /foo+\sbar\S/
 #  139|     getParsed: [RegExpSequence] foo+\sbar\S
-#  139|       0: [RegExpConstant, RegExpNormalChar] f
-#  139|       1: [RegExpConstant, RegExpNormalChar] o
-#  139|       2: [RegExpPlus] o+
+#  139|       0: [RegExpConstant, RegExpNormalChar] fo
+#  139|       1: [RegExpPlus] o+
 #  139|         0: [RegExpConstant, RegExpNormalChar] o
-#  139|       3: [RegExpCharacterClassEscape] \s
-#  139|       4: [RegExpConstant, RegExpNormalChar] b
-#  139|       5: [RegExpConstant, RegExpNormalChar] a
-#  139|       6: [RegExpConstant, RegExpNormalChar] r
-#  139|       7: [RegExpCharacterClassEscape] \S
+#  139|       2: [RegExpCharacterClassEscape] \s
+#  139|       3: [RegExpConstant, RegExpNormalChar] bar
+#  139|       4: [RegExpCharacterClassEscape] \S
 #  139|     getComponent: [RegExpTextComponent] foo+
 #  139|     getComponent: [RegExpEscapeSequenceComponent] \s
 #  139|     getComponent: [RegExpTextComponent] bar
 #  139|     getComponent: [RegExpEscapeSequenceComponent] \S
 #  140|   getStmt: [RegExpLiteral] /foo#{...}bar#{...}#{...}/
-#  140|     getParsed: [RegExpSequence] foo2barbarbar
-#  140|       0: [RegExpConstant, RegExpNormalChar] f
-#  140|       1: [RegExpConstant, RegExpNormalChar] o
-#  140|       2: [RegExpConstant, RegExpNormalChar] o
-#  140|       3: [RegExpConstant, RegExpNormalChar] 2
-#  140|       4: [RegExpConstant, RegExpNormalChar] b
-#  140|       5: [RegExpConstant, RegExpNormalChar] a
-#  140|       6: [RegExpConstant, RegExpNormalChar] r
-#  140|       7: [RegExpConstant, RegExpNormalChar] b
-#  140|       8: [RegExpConstant, RegExpNormalChar] a
-#  140|       9: [RegExpConstant, RegExpNormalChar] r
-#  140|       10: [RegExpConstant, RegExpNormalChar] b
-#  140|       11: [RegExpConstant, RegExpNormalChar] a
-#  140|       12: [RegExpConstant, RegExpNormalChar] r
+#  140|     getParsed: [RegExpConstant, RegExpNormalChar] foo2barbarbar
 #  140|     getComponent: [RegExpTextComponent] foo
 #  140|     getComponent: [RegExpInterpolationComponent] #{...}
 #  140|       getStmt: [AddExpr] ... + ...
@@ -1878,47 +1854,25 @@ literals/literals.rb:
 #  141|     getComponent: [RegExpTextComponent] foo
 #  142|   getStmt: [RegExpLiteral] //
 #  143|   getStmt: [RegExpLiteral] /foo/
-#  143|     getParsed: [RegExpSequence] foo
-#  143|       0: [RegExpConstant, RegExpNormalChar] f
-#  143|       1: [RegExpConstant, RegExpNormalChar] o
-#  143|       2: [RegExpConstant, RegExpNormalChar] o
+#  143|     getParsed: [RegExpConstant, RegExpNormalChar] foo
 #  143|     getComponent: [RegExpTextComponent] foo
 #  144|   getStmt: [RegExpLiteral] /foo/
-#  144|     getParsed: [RegExpSequence] foo
-#  144|       0: [RegExpConstant, RegExpNormalChar] f
-#  144|       1: [RegExpConstant, RegExpNormalChar] o
-#  144|       2: [RegExpConstant, RegExpNormalChar] o
+#  144|     getParsed: [RegExpConstant, RegExpNormalChar] foo
 #  144|     getComponent: [RegExpTextComponent] foo
 #  145|   getStmt: [RegExpLiteral] /foo+\sbar\S/
 #  145|     getParsed: [RegExpSequence] foo+\sbar\S
-#  145|       0: [RegExpConstant, RegExpNormalChar] f
-#  145|       1: [RegExpConstant, RegExpNormalChar] o
-#  145|       2: [RegExpPlus] o+
+#  145|       0: [RegExpConstant, RegExpNormalChar] fo
+#  145|       1: [RegExpPlus] o+
 #  145|         0: [RegExpConstant, RegExpNormalChar] o
-#  145|       3: [RegExpCharacterClassEscape] \s
-#  145|       4: [RegExpConstant, RegExpNormalChar] b
-#  145|       5: [RegExpConstant, RegExpNormalChar] a
-#  145|       6: [RegExpConstant, RegExpNormalChar] r
-#  145|       7: [RegExpCharacterClassEscape] \S
+#  145|       2: [RegExpCharacterClassEscape] \s
+#  145|       3: [RegExpConstant, RegExpNormalChar] bar
+#  145|       4: [RegExpCharacterClassEscape] \S
 #  145|     getComponent: [RegExpTextComponent] foo+
 #  145|     getComponent: [RegExpEscapeSequenceComponent] \s
 #  145|     getComponent: [RegExpTextComponent] bar
 #  145|     getComponent: [RegExpEscapeSequenceComponent] \S
 #  146|   getStmt: [RegExpLiteral] /foo#{...}bar#{...}#{...}/
-#  146|     getParsed: [RegExpSequence] foo2barbarbar
-#  146|       0: [RegExpConstant, RegExpNormalChar] f
-#  146|       1: [RegExpConstant, RegExpNormalChar] o
-#  146|       2: [RegExpConstant, RegExpNormalChar] o
-#  146|       3: [RegExpConstant, RegExpNormalChar] 2
-#  146|       4: [RegExpConstant, RegExpNormalChar] b
-#  146|       5: [RegExpConstant, RegExpNormalChar] a
-#  146|       6: [RegExpConstant, RegExpNormalChar] r
-#  146|       7: [RegExpConstant, RegExpNormalChar] b
-#  146|       8: [RegExpConstant, RegExpNormalChar] a
-#  146|       9: [RegExpConstant, RegExpNormalChar] r
-#  146|       10: [RegExpConstant, RegExpNormalChar] b
-#  146|       11: [RegExpConstant, RegExpNormalChar] a
-#  146|       12: [RegExpConstant, RegExpNormalChar] r
+#  146|     getParsed: [RegExpConstant, RegExpNormalChar] foo2barbarbar
 #  146|     getComponent: [RegExpTextComponent] foo
 #  146|     getComponent: [RegExpInterpolationComponent] #{...}
 #  146|       getStmt: [AddExpr] ... + ...
@@ -2469,10 +2423,8 @@ operations/operations.rb:
 #   65|     getAnOperand/getLeftOperand/getReceiver: [LocalVariableAccess] name
 #   65|     getAnOperand/getArgument/getRightOperand: [RegExpLiteral] /foo.*/
 #   65|       getParsed: [RegExpSequence] foo.*
-#   65|         0: [RegExpConstant, RegExpNormalChar] f
-#   65|         1: [RegExpConstant, RegExpNormalChar] o
-#   65|         2: [RegExpConstant, RegExpNormalChar] o
-#   65|         3: [RegExpStar] .*
+#   65|         0: [RegExpConstant, RegExpNormalChar] foo
+#   65|         1: [RegExpStar] .*
 #   65|           0: [RegExpDot] .
 #   65|       getComponent: [RegExpTextComponent] foo.*
 #   66|   getStmt: [NoRegExpMatchExpr] ... !~ ...
@@ -2481,9 +2433,7 @@ operations/operations.rb:
 #   66|       getParsed: [RegExpSequence] .*bar
 #   66|         0: [RegExpStar] .*
 #   66|           0: [RegExpDot] .
-#   66|         1: [RegExpConstant, RegExpNormalChar] b
-#   66|         2: [RegExpConstant, RegExpNormalChar] a
-#   66|         3: [RegExpConstant, RegExpNormalChar] r
+#   66|         1: [RegExpConstant, RegExpNormalChar] bar
 #   66|       getComponent: [RegExpTextComponent] .*bar
 #   69|   getStmt: [AssignAddExpr] ... += ...
 #   69|     getAnOperand/getLeftOperand: [LocalVariableAccess] x
--- a/ruby/ql/test/library-tests/regexp/parse.expected
+++ b/ruby/ql/test/library-tests/regexp/parse.expected
@@ -1,14 +1,5 @@
 regexp.rb:
-#    5| [RegExpConstant, RegExpNormalChar] a
-
-#    5| [RegExpSequence] abc
-#-----| 0 -> [RegExpConstant, RegExpNormalChar] a
-#-----| 1 -> [RegExpConstant, RegExpNormalChar] b
-#-----| 2 -> [RegExpConstant, RegExpNormalChar] c
-
-#    5| [RegExpConstant, RegExpNormalChar] b
-
-#    5| [RegExpConstant, RegExpNormalChar] c
+#    5| [RegExpConstant, RegExpNormalChar] abc

 #    8| [RegExpConstant, RegExpNormalChar] a

@@ -38,70 +29,36 @@ regexp.rb:
 #    9| [RegExpRange] a{4,8}
 #-----| 0 -> [RegExpConstant, RegExpNormalChar] a

-#    9| [RegExpNormalChar] 4
-
-#    9| [RegExpNormalChar] ,
-
-#    9| [RegExpNormalChar] 8
-
-#    9| [RegExpNormalChar] }
+#    9| [RegExpNormalChar] 4,8}

 #   10| [RegExpConstant, RegExpNormalChar] a

 #   10| [RegExpRange] a{,8}
 #-----| 0 -> [RegExpConstant, RegExpNormalChar] a

-#   10| [RegExpNormalChar] ,
-
-#   10| [RegExpNormalChar] 8
-
-#   10| [RegExpNormalChar] }
+#   10| [RegExpNormalChar] ,8}

 #   11| [RegExpConstant, RegExpNormalChar] a

 #   11| [InfiniteRepetitionQuantifier, RegExpRange] a{3,}
 #-----| 0 -> [RegExpConstant, RegExpNormalChar] a

-#   11| [RegExpNormalChar] 3
-
-#   11| [RegExpNormalChar] ,
-
-#   11| [RegExpNormalChar] }
+#   11| [RegExpNormalChar] 3,}

 #   12| [RegExpConstant, RegExpNormalChar] a

 #   12| [RegExpRange] a{7}
 #-----| 0 -> [RegExpConstant, RegExpNormalChar] a

-#   12| [RegExpNormalChar] 7
+#   12| [RegExpNormalChar] 7}

-#   12| [RegExpNormalChar] }
-
-#   15| [RegExpConstant, RegExpNormalChar] f
-
-#   15| [RegExpSequence] foo
-#-----| 0 -> [RegExpConstant, RegExpNormalChar] f
-#-----| 1 -> [RegExpConstant, RegExpNormalChar] o
-#-----| 2 -> [RegExpConstant, RegExpNormalChar] o
+#   15| [RegExpConstant, RegExpNormalChar] foo

 #   15| [RegExpAlt] foo|bar
-#-----| 0 -> [RegExpSequence] foo
-#-----| 1 -> [RegExpSequence] bar
+#-----| 0 -> [RegExpConstant, RegExpNormalChar] foo
+#-----| 1 -> [RegExpConstant, RegExpNormalChar] bar

-#   15| [RegExpConstant, RegExpNormalChar] o
-
-#   15| [RegExpConstant, RegExpNormalChar] o
-
-#   15| [RegExpConstant, RegExpNormalChar] b
-
-#   15| [RegExpSequence] bar
-#-----| 0 -> [RegExpConstant, RegExpNormalChar] b
-#-----| 1 -> [RegExpConstant, RegExpNormalChar] a
-#-----| 2 -> [RegExpConstant, RegExpNormalChar] r
-
-#   15| [RegExpConstant, RegExpNormalChar] a
-
-#   15| [RegExpConstant, RegExpNormalChar] r
+#   15| [RegExpConstant, RegExpNormalChar] bar

 #   18| [RegExpCharacterClass] [abc]
 #-----| 0 -> [RegExpConstant, RegExpNormalChar] a
@@ -229,10 +186,7 @@ regexp.rb:

 #   29| [RegExpSequence] [[a-f]A-F]
 #-----| 0 -> [RegExpCharacterClass] [[a-f]
-#-----| 1 -> [RegExpConstant, RegExpNormalChar] A
-#-----| 2 -> [RegExpConstant, RegExpNormalChar] -
-#-----| 3 -> [RegExpConstant, RegExpNormalChar] F
-#-----| 4 -> [RegExpConstant, RegExpNormalChar] ]
+#-----| 1 -> [RegExpConstant, RegExpNormalChar] A-F]

 #   29| [RegExpConstant, RegExpNormalChar] [

@@ -244,13 +198,7 @@ regexp.rb:

 #   29| [RegExpConstant, RegExpNormalChar] f

-#   29| [RegExpConstant, RegExpNormalChar] A
-
-#   29| [RegExpConstant, RegExpNormalChar] -
-
-#   29| [RegExpConstant, RegExpNormalChar] F
-
-#   29| [RegExpConstant, RegExpNormalChar] ]
+#   29| [RegExpConstant, RegExpNormalChar] A-F]

 #   32| [RegExpDot] .

@@ -312,69 +260,41 @@ regexp.rb:

 #   41| [RegExpSequence] \Gabc
 #-----| 0 -> [RegExpSpecialChar] \G
-#-----| 1 -> [RegExpConstant, RegExpNormalChar] a
-#-----| 2 -> [RegExpConstant, RegExpNormalChar] b
-#-----| 3 -> [RegExpConstant, RegExpNormalChar] c
+#-----| 1 -> [RegExpConstant, RegExpNormalChar] abc

-#   41| [RegExpConstant, RegExpNormalChar] a
-
-#   41| [RegExpConstant, RegExpNormalChar] b
-
-#   41| [RegExpConstant, RegExpNormalChar] c
+#   41| [RegExpConstant, RegExpNormalChar] abc

 #   42| [RegExpSpecialChar] \b

 #   42| [RegExpSequence] \b!a\B
 #-----| 0 -> [RegExpSpecialChar] \b
-#-----| 1 -> [RegExpConstant, RegExpNormalChar] !
-#-----| 2 -> [RegExpConstant, RegExpNormalChar] a
-#-----| 3 -> [RegExpSpecialChar] \B
+#-----| 1 -> [RegExpConstant, RegExpNormalChar] !a
+#-----| 2 -> [RegExpSpecialChar] \B

-#   42| [RegExpConstant, RegExpNormalChar] !
-
-#   42| [RegExpConstant, RegExpNormalChar] a
+#   42| [RegExpConstant, RegExpNormalChar] !a

 #   42| [RegExpSpecialChar] \B

 #   45| [RegExpGroup] (foo)
-#-----| 0 -> [RegExpSequence] foo
+#-----| 0 -> [RegExpConstant, RegExpNormalChar] foo

 #   45| [RegExpStar] (foo)*
 #-----| 0 -> [RegExpGroup] (foo)

 #   45| [RegExpSequence] (foo)*bar
 #-----| 0 -> [RegExpStar] (foo)*
-#-----| 1 -> [RegExpConstant, RegExpNormalChar] b
-#-----| 2 -> [RegExpConstant, RegExpNormalChar] a
-#-----| 3 -> [RegExpConstant, RegExpNormalChar] r
+#-----| 1 -> [RegExpConstant, RegExpNormalChar] bar

-#   45| [RegExpConstant, RegExpNormalChar] f
+#   45| [RegExpConstant, RegExpNormalChar] foo

-#   45| [RegExpSequence] foo
-#-----| 0 -> [RegExpConstant, RegExpNormalChar] f
-#-----| 1 -> [RegExpConstant, RegExpNormalChar] o
-#-----| 2 -> [RegExpConstant, RegExpNormalChar] o
+#   45| [RegExpConstant, RegExpNormalChar] bar

-#   45| [RegExpConstant, RegExpNormalChar] o
-
-#   45| [RegExpConstant, RegExpNormalChar] o
-
-#   45| [RegExpConstant, RegExpNormalChar] b
-
-#   45| [RegExpConstant, RegExpNormalChar] a
-
-#   45| [RegExpConstant, RegExpNormalChar] r
-
-#   46| [RegExpConstant, RegExpNormalChar] f
+#   46| [RegExpConstant, RegExpNormalChar] fo

 #   46| [RegExpSequence] fo(o|b)ar
-#-----| 0 -> [RegExpConstant, RegExpNormalChar] f
-#-----| 1 -> [RegExpConstant, RegExpNormalChar] o
-#-----| 2 -> [RegExpGroup] (o|b)
-#-----| 3 -> [RegExpConstant, RegExpNormalChar] a
-#-----| 4 -> [RegExpConstant, RegExpNormalChar] r
-
-#   46| [RegExpConstant, RegExpNormalChar] o
+#-----| 0 -> [RegExpConstant, RegExpNormalChar] fo
+#-----| 1 -> [RegExpGroup] (o|b)
+#-----| 2 -> [RegExpConstant, RegExpNormalChar] ar

 #   46| [RegExpGroup] (o|b)
 #-----| 0 -> [RegExpAlt] o|b
@@ -387,9 +307,7 @@ regexp.rb:

 #   46| [RegExpConstant, RegExpNormalChar] b

-#   46| [RegExpConstant, RegExpNormalChar] a
-
-#   46| [RegExpConstant, RegExpNormalChar] r
+#   46| [RegExpConstant, RegExpNormalChar] ar

 #   47| [RegExpGroup] (a|b|cd)
 #-----| 0 -> [RegExpAlt] a|b|cd
@@ -403,17 +321,11 @@ regexp.rb:
 #   47| [RegExpAlt] a|b|cd
 #-----| 0 -> [RegExpConstant, RegExpNormalChar] a
 #-----| 1 -> [RegExpConstant, RegExpNormalChar] b
-#-----| 2 -> [RegExpSequence] cd
+#-----| 2 -> [RegExpConstant, RegExpNormalChar] cd

 #   47| [RegExpConstant, RegExpNormalChar] b

-#   47| [RegExpConstant, RegExpNormalChar] c
-
-#   47| [RegExpSequence] cd
-#-----| 0 -> [RegExpConstant, RegExpNormalChar] c
-#-----| 1 -> [RegExpConstant, RegExpNormalChar] d
-
-#   47| [RegExpConstant, RegExpNormalChar] d
+#   47| [RegExpConstant, RegExpNormalChar] cd

 #   47| [RegExpConstant, RegExpNormalChar] e

@@ -511,13 +423,7 @@ regexp.rb:
 #   61| [RegExpRange] \p{^Alnum}{2,3}
 #-----| 0 -> [RegExpNamedCharacterProperty] \p{^Alnum}

-#   61| [RegExpNormalChar] 2
-
-#   61| [RegExpNormalChar] ,
-
-#   61| [RegExpNormalChar] 3
-
-#   61| [RegExpNormalChar] }
+#   61| [RegExpNormalChar] 2,3}

 #   62| [RegExpCharacterClass] [a-f\p{Digit}]
 #-----| 0 -> [RegExpCharacterRange] a-f
@@ -583,13 +489,4 @@ regexp.rb:

 #   74| [RegExpNamedCharacterProperty] [:digit:]

-#   78| [RegExpConstant, RegExpNormalChar] a
-
-#   78| [RegExpSequence] abc
-#-----| 0 -> [RegExpConstant, RegExpNormalChar] a
-#-----| 1 -> [RegExpConstant, RegExpNormalChar] b
-#-----| 2 -> [RegExpConstant, RegExpNormalChar] c
-
-#   78| [RegExpConstant, RegExpNormalChar] b
-
-#   78| [RegExpConstant, RegExpNormalChar] c
+#   78| [RegExpConstant, RegExpNormalChar] abc
--- a/ruby/ql/test/query-tests/security/cwe-1333-exponential-redos/ReDoS.expected
+++ b/ruby/ql/test/query-tests/security/cwe-1333-exponential-redos/ReDoS.expected
@@ -54,7 +54,7 @@
 | tst.rb:218:11:218:15 | [^X]+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'W'. |
 | tst.rb:221:16:221:16 | b | This part of the regular expression may cause exponential backtracking on strings starting with 'W' and containing many repetitions of 'bW'. |
 | tst.rb:227:16:227:16 | b | This part of the regular expression may cause exponential backtracking on strings starting with 'W' and containing many repetitions of 'bW'. |
-| tst.rb:239:13:239:13 | b | This part of the regular expression may cause exponential backtracking on strings starting with 'a' and containing many repetitions of 'ba'. |
+| tst.rb:239:12:239:13 | ab | This part of the regular expression may cause exponential backtracking on strings starting with 'a' and containing many repetitions of 'ab'. |
 | tst.rb:245:11:245:17 | [\\n\\s]+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '\\n'. |
 | tst.rb:254:11:254:13 | \\w* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'foobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbaz'. |
 | tst.rb:254:23:254:25 | \\w* | This part of the regular expression may cause exponential backtracking on strings starting with 'foobarbaz' and containing many repetitions of 'foobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbaz'. |