hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-28 02:05:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add parser tests; fix some parser issues.

Browse Source 
[temporarily renamed existing regex/Test.java during rebasing to avoid conflict]
This commit is contained in:
Joe Farebrother
2022-02-09 14:06:26 +00:00
parent 8e1918216e
commit 28649da187
5 changed files with 211 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
java/ql/lib/semmle/code/java/regex/regex.qll
View File

@@ -21,7 +21,7 @@ abstract class RegexString extends Expr {
private predicate char_set_end0(int pos) {
this.nonEscapedCharAt(pos) = "]" and
/* special case: `[]]` and `[^]]` are valid char classes. */
not this.char_set_start0(_, pos - 1)
not this.char_set_start0(_, pos)
}
/**
@@ -283,7 +283,7 @@ abstract class RegexString extends Expr {
*/
predicate escapedCharacter(int start, int end) {
this.escapingChar(start) and
not this.numbered_backreference(start, _, _) and
not this.backreference(start, _) and
(
// hex value \xhh
this.getChar(start + 1) = "x" and
@@ -362,7 +362,8 @@ abstract class RegexString extends Expr {
predicate character(int start, int end) {
(
this.simpleCharacter(start, end) and
not exists(int x, int y | this.escapedCharacter(x, y) and x <= start and y >= end)
not exists(int x, int y | this.escapedCharacter(x, y) and x <= start and y >= end) and
not exists(int x, int y | this.quote(x, y) and x <= start and y >= end)
or
this.escapedCharacter(start, end)
) and
@@ -486,8 +487,6 @@ abstract class RegexString extends Expr {
or
this.named_group_start(start, end)
or
this.named_backreference_start(start, end)
or
this.lookahead_assertion_start(start, end)
or
this.negative_lookahead_assertion_start(start, end)
@@ -526,16 +525,6 @@ abstract class RegexString extends Expr {
)
}
private predicate named_backreference_start(int start, int end) {
this.isGroupStart(start) and
this.getChar(start + 1) = "?" and
this.getChar(start + 2) = "k" and
this.getChar(start + 3) = "=" and
// Should this be looking for unescaped ")"?
// TODO: test this
end = min(int i | i > start + 4 and this.getChar(i) = "?")
}
private predicate flag_group_start(int start, int end, string c) {
this.isGroupStart(start) and
this.getChar(start + 1) = "?" and
@@ -609,9 +598,11 @@ abstract class RegexString extends Expr {
}
private predicate named_backreference(int start, int end, string name) {
this.named_backreference_start(start, start + 4) and
end = min(int i | i > start + 4 and this.getChar(i) = ")") + 1 and
name = this.getText().substring(start + 4, end - 2)
this.escapingChar(start) and
this.getChar(start + 1) = "k" and
this.getChar(start + 2) = "<" and
end = min(int i | i > start + 2 and this.getChar(i) = ">") + 1 and
name = this.getText().substring(start + 3, end - 2)
}
private predicate numbered_backreference(int start, int end, int value) {
@@ -660,6 +651,8 @@ abstract class RegexString extends Expr {
this.charSet(start, end)
or
this.backreference(start, end)
or
this.quote(start, end)
}
private predicate qualifier(int start, int end, boolean maybe_empty, boolean may_repeat_forever) {

68
java/ql/test/library-tests/regex/RegexParseTests.expected Normal file
View File

@@ -0,0 +1,68 @@
parseFailures
#select
| Test.java:5:10:5:16 | [A-Z\\d] | [RegExpCharacterClass] |
| Test.java:5:10:5:18 | [A-Z\\d]++ | [RegExpPlus] |
| Test.java:5:11:5:11 | A | [RegExpConstant,RegExpNormalChar] |
| Test.java:5:11:5:13 | A-Z | [RegExpCharacterRange] |
| Test.java:5:13:5:13 | Z | [RegExpConstant,RegExpNormalChar] |
| Test.java:5:14:5:15 | \\d | [RegExpCharacterClassEscape] |
| Test.java:6:10:6:39 | \\Q hello world [ *** \\Q ) ( \\E | [RegExpConstant,RegExpQuote] |
| Test.java:7:10:7:21 | [\\Q hi ] \\E] | [RegExpCharacterClass] |
| Test.java:7:11:7:20 | \\Q hi ] \\E | [RegExpConstant,RegExpQuote] |
| Test.java:8:10:8:12 | []] | [RegExpCharacterClass] |
| Test.java:8:11:8:11 | ] | [RegExpConstant,RegExpNormalChar] |
| Test.java:9:10:9:13 | [^]] | [RegExpCharacterClass] |
| Test.java:9:12:9:12 | ] | [RegExpConstant,RegExpNormalChar] |
| Test.java:10:10:10:20 | [abc[defg]] | [RegExpCharacterClass] |
| Test.java:10:11:10:11 | a | [RegExpConstant,RegExpNormalChar] |
| Test.java:10:12:10:12 | b | [RegExpConstant,RegExpNormalChar] |
| Test.java:10:13:10:13 | c | [RegExpConstant,RegExpNormalChar] |
| Test.java:10:14:10:14 | [ | [RegExpConstant,RegExpNormalChar] |
| Test.java:10:15:10:15 | d | [RegExpConstant,RegExpNormalChar] |
| Test.java:10:16:10:16 | e | [RegExpConstant,RegExpNormalChar] |
| Test.java:10:17:10:17 | f | [RegExpConstant,RegExpNormalChar] |
| Test.java:10:18:10:18 | g | [RegExpConstant,RegExpNormalChar] |
| Test.java:10:19:10:19 | ] | [RegExpConstant,RegExpNormalChar] |
| Test.java:11:10:11:53 | [abc&&[\\W\\p{Lower}\\P{Space}\\N{degree sign}]] | [RegExpCharacterClass] |
| Test.java:11:10:11:62 | [abc&&[\\W\\p{Lower}\\P{Space}\\N{degree sign}]]\\b7\\b{g}8 | [RegExpSequence] |
| Test.java:11:11:11:11 | a | [RegExpConstant,RegExpNormalChar] |
| Test.java:11:12:11:12 | b | [RegExpConstant,RegExpNormalChar] |
| Test.java:11:13:11:13 | c | [RegExpConstant,RegExpNormalChar] |
| Test.java:11:14:11:14 | & | [RegExpConstant,RegExpNormalChar] |
| Test.java:11:15:11:15 | & | [RegExpConstant,RegExpNormalChar] |
| Test.java:11:16:11:16 | [ | [RegExpConstant,RegExpNormalChar] |
| Test.java:11:17:11:18 | \\W | [RegExpCharacterClassEscape] |
| Test.java:11:19:11:27 | \\p{Lower} | [RegExpCharacterClassEscape] |
| Test.java:11:28:11:36 | \\P{Space} | [RegExpCharacterClassEscape] |
| Test.java:11:37:11:51 | \\N{degree sign} | [RegExpConstant,RegExpEscape] |
| Test.java:11:52:11:52 | ] | [RegExpConstant,RegExpNormalChar] |
| Test.java:11:54:11:55 | \\b | [RegExpConstant,RegExpEscape] |
| Test.java:11:56:11:56 | 7 | [RegExpConstant,RegExpNormalChar] |
| Test.java:11:57:11:61 | \\b{g} | [RegExpConstant,RegExpEscape] |
| Test.java:11:62:11:62 | 8 | [RegExpConstant,RegExpNormalChar] |
| Test.java:12:10:12:12 | \\cA | [RegExpConstant,RegExpEscape] |
| Test.java:13:10:13:12 | \\c( | [RegExpConstant,RegExpEscape] |
| Test.java:14:10:14:12 | \\c\\ | [RegExpConstant,RegExpEscape] |
| Test.java:14:10:14:16 | \\c\\(ab) | [RegExpSequence] |
| Test.java:14:13:14:16 | (ab) | [RegExpGroup] |
| Test.java:14:14:14:14 | a | [RegExpConstant,RegExpNormalChar] |
| Test.java:14:14:14:15 | ab | [RegExpSequence] |
| Test.java:14:15:14:15 | b | [RegExpConstant,RegExpNormalChar] |
| Test.java:15:10:15:15 | (?>hi) | [RegExpGroup] |
| Test.java:15:10:15:44 | (?>hi)(?<name>hell*?o*+)123\\k<name> | [RegExpSequence] |
| Test.java:15:13:15:13 | h | [RegExpConstant,RegExpNormalChar] |
| Test.java:15:13:15:14 | hi | [RegExpSequence] |
| Test.java:15:14:15:14 | i | [RegExpConstant,RegExpNormalChar] |
| Test.java:15:16:15:33 | (?<name>hell*?o*+) | [RegExpGroup] |
| Test.java:15:24:15:24 | h | [RegExpConstant,RegExpNormalChar] |
| Test.java:15:24:15:32 | hell*?o*+ | [RegExpSequence] |
| Test.java:15:25:15:25 | e | [RegExpConstant,RegExpNormalChar] |
| Test.java:15:26:15:26 | l | [RegExpConstant,RegExpNormalChar] |
| Test.java:15:27:15:27 | l | [RegExpConstant,RegExpNormalChar] |
| Test.java:15:27:15:29 | l*? | [RegExpStar] |
| Test.java:15:30:15:30 | o | [RegExpConstant,RegExpNormalChar] |
| Test.java:15:30:15:32 | o*+ | [RegExpStar] |
| Test.java:15:34:15:34 | 1 | [RegExpConstant,RegExpNormalChar] |
| Test.java:15:35:15:35 | 2 | [RegExpConstant,RegExpNormalChar] |
| Test.java:15:36:15:36 | 3 | [RegExpConstant,RegExpNormalChar] |
| Test.java:15:37:15:44 | \\k<name> | [RegExpBackRef] |

10
java/ql/test/library-tests/regex/RegexParseTests.ql Normal file
View File

@@ -0,0 +1,10 @@
import java
import semmle.code.java.regex.RegexTreeView
import semmle.code.java.regex.regex
string getQLClases(RegExpTerm t) { result = "[" + strictconcat(t.getPrimaryQLClass(), ",") + "]" }
query predicate parseFailures(Regex r, int i) { r.failedToParse(i) }
from RegExpTerm t
select t, getQLClases(t)

117
java/ql/test/library-tests/regex/Test.java
View File

@@ -1,104 +1,23 @@
package generatedtest;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
// Test case generated by GenerateFlowTestCase.ql
public class Test {
class Test {
static String[] regs = {
"[A-Z\\d]++",
"\\Q hello world [ *** \\Q ) ( \\E",
"[\\Q hi ] \\E]",
"[]]",
"[^]]",
"[abc[defg]]",
"[abc&&[\\W\\p{Lower}\\P{Space}\\N{degree sign}]]\\b7\\b{g}8",
"\\cA",
"\\c(",
"\\c\\(ab)",
"(?>hi)(?<name>hell*?o*+)123\\k<name>"
};
private final String str_pattern = "\\$\\{(.*)\\}";
private final Pattern pattern = Pattern.compile(str_pattern);
Object source() { return null; }
void sink(Object o) { }
public void test() throws Exception {
{
// "java.util.regex;Matcher;false;group;;;Argument[-1];ReturnValue;taint"
String out = null;
String in = (String) source();
Matcher m = pattern.matcher(in);
out = m.group("foo");
sink(out); // $ hasTaintFlow
void test() {
for (int i = 0; i < regs.length; i++) {
Pattern.compile(regs[i]);
}
}
{
// "java.util.regex;Matcher;false;group;;;Argument[-1];ReturnValue;taint"
String out = null;
String in = (String) source();
Matcher m = pattern.matcher(in);
out = m.group();
sink(out); // $ hasTaintFlow
}
{
// "java.util.regex;Matcher;false;group;;;Argument[-1];ReturnValue;taint"
String out = null;
String in = (String) source();
Matcher m = pattern.matcher(in);
out = m.group(0);
sink(out); // $ hasTaintFlow
}
{
// "java.util.regex;Matcher;false;replaceAll;;;Argument[-1];ReturnValue;taint"
String out = null;
String in = (String) source();
Matcher m = pattern.matcher(in);
out = m.replaceAll("foo");
sink(out); // $ hasTaintFlow
}
{
// "java.util.regex;Matcher;false;replaceAll;;;Argument[0];ReturnValue;taint"
String out = null;
String in = (String) source();
Matcher m = pattern.matcher("foo");
out = m.replaceAll(in);
sink(out); // $ hasTaintFlow
}
{
// "java.util.regex;Matcher;false;replaceFirst;;;Argument[-1];ReturnValue;taint"
String out = null;
String in = (String) source();
Matcher m = pattern.matcher(in);
out = m.replaceFirst("foo");
sink(out); // $ hasTaintFlow
}
{
// "java.util.regex;Matcher;false;replaceFirst;;;Argument[0];ReturnValue;taint"
String out = null;
String in = (String) source();
Matcher m = pattern.matcher("foo");
out = m.replaceFirst(in);
sink(out); // $ hasTaintFlow
}
{
// "java.util.regex;Pattern;false;matcher;;;Argument[0];ReturnValue;taint"
Matcher out = null;
CharSequence in = (CharSequence)source();
out = pattern.matcher(in);
sink(out); // $ hasTaintFlow
}
{
// "java.util.regex;Pattern;false;quote;;;Argument[0];ReturnValue;taint"
String out = null;
String in = (String)source();
out = Pattern.quote(in);
sink(out); // $ hasTaintFlow
}
{
// "java.util.regex;Pattern;false;split;;;Argument[0];ReturnValue;taint"
String[] out = null;
CharSequence in = (CharSequence)source();
out = pattern.split(in);
sink(out); // $ hasTaintFlow
}
{
// "java.util.regex;Pattern;false;split;;;Argument[0];ReturnValue;taint"
String[] out = null;
CharSequence in = (CharSequence)source();
out = pattern.split(in, 0);
sink(out); // $ hasTaintFlow
}
}
}

104
java/ql/test/library-tests/regex/Test2.java Normal file
View File

@@ -0,0 +1,104 @@
package generatedtest;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
// Test case generated by GenerateFlowTestCase.ql
public class Test {
private final String str_pattern = "\\$\\{(.*)\\}";
private final Pattern pattern = Pattern.compile(str_pattern);
Object source() { return null; }
void sink(Object o) { }
public void test() throws Exception {
{
// "java.util.regex;Matcher;false;group;;;Argument[-1];ReturnValue;taint"
String out = null;
String in = (String) source();
Matcher m = pattern.matcher(in);
out = m.group("foo");
sink(out); // $ hasTaintFlow
}
{
// "java.util.regex;Matcher;false;group;;;Argument[-1];ReturnValue;taint"
String out = null;
String in = (String) source();
Matcher m = pattern.matcher(in);
out = m.group();
sink(out); // $ hasTaintFlow
}
{
// "java.util.regex;Matcher;false;group;;;Argument[-1];ReturnValue;taint"
String out = null;
String in = (String) source();
Matcher m = pattern.matcher(in);
out = m.group(0);
sink(out); // $ hasTaintFlow
}
{
// "java.util.regex;Matcher;false;replaceAll;;;Argument[-1];ReturnValue;taint"
String out = null;
String in = (String) source();
Matcher m = pattern.matcher(in);
out = m.replaceAll("foo");
sink(out); // $ hasTaintFlow
}
{
// "java.util.regex;Matcher;false;replaceAll;;;Argument[0];ReturnValue;taint"
String out = null;
String in = (String) source();
Matcher m = pattern.matcher("foo");
out = m.replaceAll(in);
sink(out); // $ hasTaintFlow
}
{
// "java.util.regex;Matcher;false;replaceFirst;;;Argument[-1];ReturnValue;taint"
String out = null;
String in = (String) source();
Matcher m = pattern.matcher(in);
out = m.replaceFirst("foo");
sink(out); // $ hasTaintFlow
}
{
// "java.util.regex;Matcher;false;replaceFirst;;;Argument[0];ReturnValue;taint"
String out = null;
String in = (String) source();
Matcher m = pattern.matcher("foo");
out = m.replaceFirst(in);
sink(out); // $ hasTaintFlow
}
{
// "java.util.regex;Pattern;false;matcher;;;Argument[0];ReturnValue;taint"
Matcher out = null;
CharSequence in = (CharSequence)source();
out = pattern.matcher(in);
sink(out); // $ hasTaintFlow
}
{
// "java.util.regex;Pattern;false;quote;;;Argument[0];ReturnValue;taint"
String out = null;
String in = (String)source();
out = Pattern.quote(in);
sink(out); // $ hasTaintFlow
}
{
// "java.util.regex;Pattern;false;split;;;Argument[0];ReturnValue;taint"
String[] out = null;
CharSequence in = (CharSequence)source();
out = pattern.split(in);
sink(out); // $ hasTaintFlow
}
{
// "java.util.regex;Pattern;false;split;;;Argument[0];ReturnValue;taint"
String[] out = null;
CharSequence in = (CharSequence)source();
out = pattern.split(in, 0);
sink(out); // $ hasTaintFlow
}
}
}