hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-07-20 18:58:36 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #7917 from aibaars/incomplete-hostname

Browse Source 
Ruby: IncompleteHostnameRegExp.ql
This commit is contained in:
Arthur Baars
2022-03-18 16:00:09 +01:00
committed by GitHub
parent 767453520e 4a27928728
commit 117fb5be7d
24 changed files with 564 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
config/identical-files.json
View File

@@ -516,8 +516,12 @@
"javascript/ql/lib/semmle/javascript/frameworks/data/internal/AccessPathSyntax.qll",
"ruby/ql/lib/codeql/ruby/dataflow/internal/AccessPathSyntax.qll"
],
"Hostname Regexp queries": [
"javascript/ql/src/Security/CWE-020/HostnameRegexpShared.qll",
"ruby/ql/src/queries/security/cwe-020/HostnameRegexpShared.qll"
],
"ApiGraphModels": [
"javascript/ql/lib/semmle/javascript/frameworks/data/internal/ApiGraphModels.qll",
"ruby/ql/lib/codeql/ruby/frameworks/data/internal/ApiGraphModels.qll"
]
}
}

10
javascript/ql/lib/semmle/javascript/Regexp.qll
View File

@@ -990,16 +990,22 @@ predicate isInterpretedAsRegExp(DataFlow::Node source) {
}
/**
* Provides regular expression patterns.
* Provides utility predicates related to regular expressions.
*/
module RegExpPatterns {
/**
* Gets a pattern that matches common top-level domain names in lower case.
*/
string commonTLD() {
string getACommonTld() {
// according to ranking by http://google.com/search?q=site:.<<TLD>>
result = "(?:com|org|edu|gov|uk|net|io)(?![a-z0-9])"
}
/**
* Gets a pattern that matches common top-level domain names in lower case.
* DEPRECATED: use `getACommonTld` instead
*/
deprecated predicate commonTLD = getACommonTld/0;
}
/**

97
javascript/ql/src/Security/CWE-020/HostnameRegexpShared.qll
View File

@@ -3,7 +3,7 @@
* that match URLs and hostname patterns.
*/
import javascript
private import HostnameRegexpSpecific
/**
* Holds if the given constant is unlikely to occur in the origin part of a URL.
@@ -62,7 +62,7 @@ predicate hasTopLevelDomainEnding(RegExpSequence seq, int i) {
seq.getChild(i)
.(RegExpConstant)
.getValue()
.regexpMatch("(?i)" + RegExpPatterns::commonTLD() + "(:\\d+)?([/?#].*)?") and
.regexpMatch("(?i)" + RegExpPatterns::getACommonTld() + "(:\\d+)?([/?#].*)?") and
isDotLike(seq.getChild(i - 1)) and
not (i = 1 and matchesBeginningOfString(seq))
}
@@ -107,3 +107,96 @@ predicate alwaysMatchesHostnameAlt(RegExpAlt alt, int i) {
alwaysMatchesHostnameAlt(alt, i - 1) and
alwaysMatchesHostname(alt.getChild(i))
}
/**
* Holds if `term` occurs inside a quantifier or alternative (and thus
* can not be expected to correspond to a unique match), or as part of
* a lookaround assertion (which are rarely used for capture groups).
*/
predicate isInsideChoiceOrSubPattern(RegExpTerm term) {
exists(RegExpParent parent | parent = term.getParent() |
parent instanceof RegExpAlt
or
parent instanceof RegExpQuantifier
or
parent instanceof RegExpSubPattern
or
isInsideChoiceOrSubPattern(parent)
)
}
/**
* Holds if `group` is likely to be used as a capture group.
*/
predicate isLikelyCaptureGroup(RegExpGroup group) {
group.isCapture() and
not isInsideChoiceOrSubPattern(group)
}
/**
* Holds if `seq` contains two consecutive dots `..` or escaped dots.
*
* At least one of these dots is not intended to be a subdomain separator,
* so we avoid flagging the pattern in this case.
*/
predicate hasConsecutiveDots(RegExpSequence seq) {
exists(int i |
isDotLike(seq.getChild(i)) and
isDotLike(seq.getChild(i + 1))
)
}
predicate isIncompleteHostNameRegExpPattern(RegExpTerm regexp, RegExpSequence seq, string msg) {
seq = regexp.getAChild*() and
exists(RegExpDot unescapedDot, int i, string hostname |
hasTopLevelDomainEnding(seq, i) and
not isConstantInvalidInsideOrigin(seq.getChild([0 .. i - 1]).getAChild*()) and
not isLikelyCaptureGroup(seq.getChild([i .. seq.getNumChild() - 1]).getAChild*()) and
unescapedDot = seq.getChild([0 .. i - 1]).getAChild*() and
unescapedDot != seq.getChild(i - 1) and // Should not be the '.' immediately before the TLD
not hasConsecutiveDots(unescapedDot.getParent()) and
hostname =
seq.getChild(i - 2).getRawValue() + seq.getChild(i - 1).getRawValue() +
seq.getChild(i).getRawValue()
|
if unescapedDot.getParent() instanceof RegExpQuantifier
then
// `.*\.example.com` can match `evil.com/?x=.example.com`
//
// This problem only occurs when the pattern is applied against a full URL, not just a hostname/origin.
// We therefore check if the pattern includes a suffix after the TLD, such as `.*\.example.com/`.
// Note that a post-anchored pattern (`.*\.example.com$`) will usually fail to match a full URL,
// and patterns with neither a suffix nor an anchor fall under the purview of MissingRegExpAnchor.
seq.getChild(0) instanceof RegExpCaret and
not seq.getAChild() instanceof RegExpDollar and
seq.getChild([i .. i + 1]).(RegExpConstant).getValue().regexpMatch(".*[/?#].*") and
msg =
"has an unrestricted wildcard '" + unescapedDot.getParent().(RegExpQuantifier).getRawValue()
+ "' which may cause '" + hostname +
"' to be matched anywhere in the URL, outside the hostname."
else
msg =
"has an unescaped '.' before '" + hostname +
"', so it might match more hosts than expected."
)
}
predicate incompleteHostnameRegExp(
RegExpSequence hostSequence, string message, DataFlow::Node aux, string label
) {
exists(RegExpPatternSource re, RegExpTerm regexp, string msg, string kind |
regexp = re.getRegExpTerm() and
isIncompleteHostNameRegExpPattern(regexp, hostSequence, msg) and
(
if re.getAParse() != re
then (
kind = "string, which is used as a regular expression $@," and
aux = re.getAParse()
) else (
kind = "regular expression" and aux = re
)
)
|
message = "This " + kind + " " + msg and label = "here"
)
}

1
javascript/ql/src/Security/CWE-020/HostnameRegexpSpecific.qll Normal file
View File

@@ -0,0 +1 @@
import javascript

4
javascript/ql/src/Security/CWE-020/IncompleteHostnameRegExp.qhelp
View File

@@ -30,7 +30,7 @@
<p>
Escape all meta-characters appropriately when constructing
regular expressions for security checks, pay special attention to the
regular expressions for security checks, and pay special attention to the
<code>.</code> meta-character.
</p>
@@ -59,7 +59,7 @@
<p>
Address this vulnerability by escaping <code>.</code>
appropriately: <code>let regex = /((www|beta)\.)?example\.com/</code>.
appropriately: <code>let regex = /^((www|beta)\.)?example\.com/</code>.
</p>

93
javascript/ql/src/Security/CWE-020/IncompleteHostnameRegExp.ql
View File

@@ -11,97 +11,6 @@
* external/cwe/cwe-020
*/
import javascript
import semmle.javascript.CharacterEscapes
import HostnameRegexpShared
/**
* Holds if `term` occurs inside a quantifier or alternative (and thus
* can not be expected to correspond to a unique match), or as part of
* a lookaround assertion (which are rarely used for capture groups).
*/
predicate isInsideChoiceOrSubPattern(RegExpTerm term) {
exists(RegExpParent parent | parent = term.getParent() |
parent instanceof RegExpAlt
or
parent instanceof RegExpQuantifier
or
parent instanceof RegExpSubPattern
or
isInsideChoiceOrSubPattern(parent)
)
}
/**
* Holds if `group` is likely to be used as a capture group.
*/
predicate isLikelyCaptureGroup(RegExpGroup group) {
group.isCapture() and
not isInsideChoiceOrSubPattern(group)
}
/**
* Holds if `seq` contains two consecutive dots `..` or escaped dots.
*
* At least one of these dots is not intended to be a subdomain separator,
* so we avoid flagging the pattern in this case.
*/
predicate hasConsecutiveDots(RegExpSequence seq) {
exists(int i |
isDotLike(seq.getChild(i)) and
isDotLike(seq.getChild(i + 1))
)
}
predicate isIncompleteHostNameRegExpPattern(RegExpTerm regexp, RegExpSequence seq, string msg) {
seq = regexp.getAChild*() and
exists(RegExpDot unescapedDot, int i, string hostname |
hasTopLevelDomainEnding(seq, i) and
not isConstantInvalidInsideOrigin(seq.getChild([0 .. i - 1]).getAChild*()) and
not isLikelyCaptureGroup(seq.getChild([i .. seq.getNumChild() - 1]).getAChild*()) and
unescapedDot = seq.getChild([0 .. i - 1]).getAChild*() and
unescapedDot != seq.getChild(i - 1) and // Should not be the '.' immediately before the TLD
not hasConsecutiveDots(unescapedDot.getParent()) and
hostname =
seq.getChild(i - 2).getRawValue() + seq.getChild(i - 1).getRawValue() +
seq.getChild(i).getRawValue()
|
if unescapedDot.getParent() instanceof RegExpQuantifier
then
// `.*\.example.com` can match `evil.com/?x=.example.com`
//
// This problem only occurs when the pattern is applied against a full URL, not just a hostname/origin.
// We therefore check if the pattern includes a suffix after the TLD, such as `.*\.example.com/`.
// Note that a post-anchored pattern (`.*\.example.com$`) will usually fail to match a full URL,
// and patterns with neither a suffix nor an anchor fall under the purview of MissingRegExpAnchor.
seq.getChild(0) instanceof RegExpCaret and
not seq.getAChild() instanceof RegExpDollar and
seq.getChild([i .. i + 1]).(RegExpConstant).getValue().regexpMatch(".*[/?#].*") and
msg =
"has an unrestricted wildcard '" + unescapedDot.getParent().(RegExpQuantifier).getRawValue()
+ "' which may cause '" + hostname +
"' to be matched anywhere in the URL, outside the hostname."
else
msg =
"has an unescaped '.' before '" + hostname +
"', so it might match more hosts than expected."
)
}
from
RegExpPatternSource re, RegExpTerm regexp, RegExpSequence hostSequence, string msg, string kind,
DataFlow::Node aux
where
regexp = re.getRegExpTerm() and
isIncompleteHostNameRegExpPattern(regexp, hostSequence, msg) and
(
if re.getAParse() != re
then (
kind = "string, which is used as a regular expression $@," and
aux = re.getAParse()
) else (
kind = "regular expression" and aux = re
)
) and
not CharacterEscapes::hasALikelyRegExpPatternMistake(re)
select hostSequence, "This " + kind + " " + msg, aux, "here"
query predicate problems = incompleteHostnameRegExp/4;

2
javascript/ql/src/Security/CWE-020/IncompleteUrlSubstringSanitization.ql
View File

@@ -39,7 +39,7 @@ where
(
// target contains a domain on a common TLD, and perhaps some other URL components
target
.regexpMatch("(?i)([a-z]*:?//)?\\.?([a-z0-9-]+\\.)+" + RegExpPatterns::commonTLD() +
.regexpMatch("(?i)([a-z]*:?//)?\\.?([a-z0-9-]+\\.)+" + RegExpPatterns::getACommonTld() +
"(:[0-9]+)?/?")
or
// target is a HTTP URL to a domain on any TLD

1
javascript/ql/test/query-tests/Security/CWE-020/IncompleteHostnameRegExp.expected
View File

@@ -23,4 +23,5 @@
| tst-IncompleteHostnameRegExp.js:48:42:48:47 | ^https?://.+.example\\.com/ | This regular expression has an unescaped '.' before 'example\\.com/', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:48:13:48:69 | '^http: ... \\.com/' | here |
| tst-IncompleteHostnameRegExp.js:48:42:48:47 | ^https?://.+.example\\.com/ | This regular expression has an unrestricted wildcard '.+' which may cause 'example\\.com/' to be matched anywhere in the URL, outside the hostname. | tst-IncompleteHostnameRegExp.js:48:13:48:69 | '^http: ... \\.com/' | here |
| tst-IncompleteHostnameRegExp.js:53:14:53:35 | test.example.com$ | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:53:13:53:36 | 'test.' ... e.com$' | here |
| tst-IncompleteHostnameRegExp.js:55:14:55:38 | ^http://test.example.com | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:55:13:55:39 | '^http: ... le.com' | here |
| tst-IncompleteHostnameRegExp.js:59:5:59:20 | foo.example\\.com | This regular expression has an unescaped '.' before 'example\\.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.js:59:2:59:32 | /^(foo. ... ever)$/ | here |

2
javascript/ql/test/query-tests/Security/CWE-020/tst-IncompleteHostnameRegExp.js
View File

@@ -52,7 +52,7 @@
new RegExp('test.' + 'example.com$'); // NOT OK
new RegExp('^http://test\.example.com'); // NOT OK, but flagged by js/useless-regexp-character-escape
new RegExp('^http://test\.example.com'); // NOT OK
/^http:\/\/(..|...)\.example\.com\/index\.html/; // OK, wildcards are intentional
/^http:\/\/.\.example\.com\/index\.html/; // OK, the wildcard is intentional

2
python/ql/src/Security/CWE-020/IncompleteHostnameRegExp.qhelp
View File

@@ -30,7 +30,7 @@
<p>
Escape all meta-characters appropriately when constructing
regular expressions for security checks, pay special attention to the
regular expressions for security checks, and pay special attention to the
<code>.</code> meta-character.
</p>

3
ruby/ql/lib/codeql/ruby/security/performance/ParseRegExp.qll
View File

@@ -480,6 +480,7 @@ abstract class RegExp extends AST::StringlikeLiteral {
/** Gets the number of the group in start,end */
int getGroupNumber(int start, int end) {
this.group(start, end) and
not this.nonCapturingGroupStart(start, _) and
result =
count(int i | this.group(i, _) and i < start and not this.nonCapturingGroupStart(i, _)) + 1
}
@@ -583,7 +584,7 @@ abstract class RegExp extends AST::StringlikeLiteral {
private predicate nonCapturingGroupStart(int start, int end) {
this.isGroupStart(start) and
this.getChar(start + 1) = "?" and
this.getChar(start + 2) = ":" and
this.getChar(start + 2) = [":", "=", "<", "!", "#"] and
end = start + 3
}

18
ruby/ql/lib/codeql/ruby/security/performance/RegExpTreeView.qll
View File

@@ -60,6 +60,19 @@ module RegExpFlags {
}
}
/**
* Provides utility predicates related to regular expressions.
*/
module RegExpPatterns {
/**
* Gets a pattern that matches common top-level domain names in lower case.
*/
string getACommonTld() {
// according to ranking by http://google.com/search?q=site:.<<TLD>>
result = "(?:com|org|edu|gov|uk|net|io)(?![a-z0-9])"
}
}
/**
* An element containing a regular expression term, that is, either
* a string literal (parsed as a regular expression)
@@ -385,6 +398,8 @@ class RegExpAlt extends RegExpTerm, TRegExpAlt {
override string getAPrimaryQlClass() { result = "RegExpAlt" }
}
class RegExpCharEscape = RegExpEscape;
class RegExpEscape extends RegExpNormalChar {
RegExpEscape() { re.escapedCharacter(start, end) }
@@ -593,6 +608,9 @@ class RegExpGroup extends RegExpTerm, TRegExpGroup {
*/
int getNumber() { result = re.getGroupNumber(start, end) }
/** Holds if this is a capture group. */
predicate isCapture() { exists(this.getNumber()) }
/** Holds if this is a named capture group. */
predicate isNamed() { exists(this.getName()) }

4
ruby/ql/src/change-notes/2022-02-10-incomplete-hostname-regexp.md Normal file
View File

@@ -0,0 +1,4 @@
---
category: newQuery
---
* Added a new query, `rb/incomplete-hostname-regexp`. The query finds instances where a hostname is incompletely sanitized due to an unescaped character in a regular expression.

202
ruby/ql/src/queries/security/cwe-020/HostnameRegexpShared.qll Normal file
View File

@@ -0,0 +1,202 @@
/**
* Provides predicates for reasoning about regular expressions
* that match URLs and hostname patterns.
*/
private import HostnameRegexpSpecific
/**
* Holds if the given constant is unlikely to occur in the origin part of a URL.
*/
predicate isConstantInvalidInsideOrigin(RegExpConstant term) {
// Look for any of these cases:
// - A character that can't occur in the origin
// - Two dashes in a row
// - A colon that is not part of port or scheme separator
// - A slash that is not part of scheme separator
term.getValue().regexpMatch(".*(?:[^a-zA-Z0-9.:/-]|--|:[^0-9/]|(?<![/:]|^)/).*")
}
/** Holds if `term` is a dot constant of form `\.` or `[.]`. */
predicate isDotConstant(RegExpTerm term) {
term.(RegExpCharEscape).getValue() = "."
or
exists(RegExpCharacterClass cls |
term = cls and
not cls.isInverted() and
cls.getNumChild() = 1 and
cls.getAChild().(RegExpConstant).getValue() = "."
)
}
/** Holds if `term` is a wildcard `.` or an actual `.` character. */
predicate isDotLike(RegExpTerm term) {
term instanceof RegExpDot
or
isDotConstant(term)
}
/** Holds if `term` will only ever be matched against the beginning of the input. */
predicate matchesBeginningOfString(RegExpTerm term) {
term.isRootTerm()
or
exists(RegExpTerm parent | matchesBeginningOfString(parent) |
term = parent.(RegExpSequence).getChild(0)
or
parent.(RegExpSequence).getChild(0) instanceof RegExpCaret and
term = parent.(RegExpSequence).getChild(1)
or
term = parent.(RegExpAlt).getAChild()
or
term = parent.(RegExpGroup).getAChild()
)
}
/**
* Holds if the given sequence contains top-level domain preceded by a dot, such as `.com`,
* excluding cases where this is at the very beginning of the regexp.
*
* `i` is bound to the index of the last child in the top-level domain part.
*/
predicate hasTopLevelDomainEnding(RegExpSequence seq, int i) {
seq.getChild(i)
.(RegExpConstant)
.getValue()
.regexpMatch("(?i)" + RegExpPatterns::getACommonTld() + "(:\\d+)?([/?#].*)?") and
isDotLike(seq.getChild(i - 1)) and
not (i = 1 and matchesBeginningOfString(seq))
}
/**
* Holds if the given regular expression term contains top-level domain preceded by a dot,
* such as `.com`.
*/
predicate hasTopLevelDomainEnding(RegExpSequence seq) { hasTopLevelDomainEnding(seq, _) }
/**
* Holds if `term` will always match a hostname, that is, all disjunctions contain
* a hostname pattern that isn't inside a quantifier.
*/
predicate alwaysMatchesHostname(RegExpTerm term) {
hasTopLevelDomainEnding(term, _)
or
// `localhost` is considered a hostname pattern, but has no TLD
term.(RegExpConstant).getValue().regexpMatch("\\blocalhost\\b")
or
not term instanceof RegExpAlt and
not term instanceof RegExpQuantifier and
alwaysMatchesHostname(term.getAChild())
or
alwaysMatchesHostnameAlt(term)
}
/** Holds if every child of `alt` contains a hostname pattern. */
predicate alwaysMatchesHostnameAlt(RegExpAlt alt) {
alwaysMatchesHostnameAlt(alt, alt.getNumChild() - 1)
}
/**
* Holds if the first `i` children of `alt` contains a hostname pattern.
*
* This is used instead of `forall` to avoid materializing the set of alternatives
* that don't contains hostnames, which is much larger.
*/
predicate alwaysMatchesHostnameAlt(RegExpAlt alt, int i) {
alwaysMatchesHostname(alt.getChild(0)) and i = 0
or
alwaysMatchesHostnameAlt(alt, i - 1) and
alwaysMatchesHostname(alt.getChild(i))
}
/**
* Holds if `term` occurs inside a quantifier or alternative (and thus
* can not be expected to correspond to a unique match), or as part of
* a lookaround assertion (which are rarely used for capture groups).
*/
predicate isInsideChoiceOrSubPattern(RegExpTerm term) {
exists(RegExpParent parent | parent = term.getParent() |
parent instanceof RegExpAlt
or
parent instanceof RegExpQuantifier
or
parent instanceof RegExpSubPattern
or
isInsideChoiceOrSubPattern(parent)
)
}
/**
* Holds if `group` is likely to be used as a capture group.
*/
predicate isLikelyCaptureGroup(RegExpGroup group) {
group.isCapture() and
not isInsideChoiceOrSubPattern(group)
}
/**
* Holds if `seq` contains two consecutive dots `..` or escaped dots.
*
* At least one of these dots is not intended to be a subdomain separator,
* so we avoid flagging the pattern in this case.
*/
predicate hasConsecutiveDots(RegExpSequence seq) {
exists(int i |
isDotLike(seq.getChild(i)) and
isDotLike(seq.getChild(i + 1))
)
}
predicate isIncompleteHostNameRegExpPattern(RegExpTerm regexp, RegExpSequence seq, string msg) {
seq = regexp.getAChild*() and
exists(RegExpDot unescapedDot, int i, string hostname |
hasTopLevelDomainEnding(seq, i) and
not isConstantInvalidInsideOrigin(seq.getChild([0 .. i - 1]).getAChild*()) and
not isLikelyCaptureGroup(seq.getChild([i .. seq.getNumChild() - 1]).getAChild*()) and
unescapedDot = seq.getChild([0 .. i - 1]).getAChild*() and
unescapedDot != seq.getChild(i - 1) and // Should not be the '.' immediately before the TLD
not hasConsecutiveDots(unescapedDot.getParent()) and
hostname =
seq.getChild(i - 2).getRawValue() + seq.getChild(i - 1).getRawValue() +
seq.getChild(i).getRawValue()
|
if unescapedDot.getParent() instanceof RegExpQuantifier
then
// `.*\.example.com` can match `evil.com/?x=.example.com`
//
// This problem only occurs when the pattern is applied against a full URL, not just a hostname/origin.
// We therefore check if the pattern includes a suffix after the TLD, such as `.*\.example.com/`.
// Note that a post-anchored pattern (`.*\.example.com$`) will usually fail to match a full URL,
// and patterns with neither a suffix nor an anchor fall under the purview of MissingRegExpAnchor.
seq.getChild(0) instanceof RegExpCaret and
not seq.getAChild() instanceof RegExpDollar and
seq.getChild([i .. i + 1]).(RegExpConstant).getValue().regexpMatch(".*[/?#].*") and
msg =
"has an unrestricted wildcard '" + unescapedDot.getParent().(RegExpQuantifier).getRawValue()
+ "' which may cause '" + hostname +
"' to be matched anywhere in the URL, outside the hostname."
else
msg =
"has an unescaped '.' before '" + hostname +
"', so it might match more hosts than expected."
)
}
predicate incompleteHostnameRegExp(
RegExpSequence hostSequence, string message, DataFlow::Node aux, string label
) {
exists(RegExpPatternSource re, RegExpTerm regexp, string msg, string kind |
regexp = re.getRegExpTerm() and
isIncompleteHostNameRegExpPattern(regexp, hostSequence, msg) and
(
if re.getAParse() != re
then (
kind = "string, which is used as a regular expression $@," and
aux = re.getAParse()
) else (
kind = "regular expression" and aux = re
)
)
|
message = "This " + kind + " " + msg and label = "here"
)
}

2
ruby/ql/src/queries/security/cwe-020/HostnameRegexpSpecific.qll Normal file
View File

@@ -0,0 +1,2 @@
import codeql.ruby.security.performance.RegExpTreeView
import codeql.ruby.DataFlow

72
ruby/ql/src/queries/security/cwe-020/IncompleteHostnameRegExp.qhelp Normal file
View File

@@ -0,0 +1,72 @@
<!DOCTYPE qhelp PUBLIC
"-//Semmle//qhelp//EN"
"qhelp.dtd">
<qhelp>
<overview>
<p>
Sanitizing untrusted URLs is an important technique for
preventing attacks such as request forgeries and malicious
redirections. Often, this is done by checking that the host of a URL
is in a set of allowed hosts.
</p>
<p>
If a regular expression implements such a check, it is
easy to accidentally make the check too permissive by not escaping the
<code>.</code> meta-characters appropriately.
Even if the check is not used in a security-critical
context, the incomplete check may still cause undesirable behaviors
when it accidentally succeeds.
</p>
</overview>
<recommendation>
<p>
Escape all meta-characters appropriately when constructing
regular expressions for security checks, and pay special attention to the
<code>.</code> meta-character.
</p>
</recommendation>
<example>
<p>
The following example code checks that a URL redirection
will reach the <code>example.com</code> domain, or one of its
subdomains.
</p>
<sample src="examples/IncompleteHostnameRegExp.rb"/>
<p>
The check is however easy to bypass because the unescaped
<code>.</code> allows for any character before
<code>example.com</code>, effectively allowing the redirect to go to
an attacker-controlled domain such as <code>wwwXexample.com</code>.
</p>
<p>
Address this vulnerability by escaping <code>.</code>
appropriately: <code>regex = /^((www|beta)\.)?example\.com/</code>.
</p>
</example>
<references>
<li>OWASP: <a href="https://www.owasp.org/index.php/Server_Side_Request_Forgery">SSRF</a></li>
<li>OWASP: <a href="https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html">XSS Unvalidated Redirects and Forwards Cheat Sheet</a>.</li>
</references>
</qhelp>

16
ruby/ql/src/queries/security/cwe-020/IncompleteHostnameRegExp.ql Normal file
View File

@@ -0,0 +1,16 @@
/**
* @name Incomplete regular expression for hostnames
* @description Matching a URL or hostname against a regular expression that contains an unescaped dot as part of the hostname might match more hostnames than expected.
* @kind problem
* @problem.severity warning
* @security-severity 7.8
* @precision high
* @id rb/incomplete-hostname-regexp
* @tags correctness
* security
* external/cwe/cwe-020
*/
import HostnameRegexpShared
query predicate problems = incompleteHostnameRegExp/4;

13
ruby/ql/src/queries/security/cwe-020/examples/IncompleteHostnameRegExp.rb Normal file
View File

@@ -0,0 +1,13 @@
class AppController < ApplicationController
def index
url = params[:url]
host = URI(url).host
# BAD: the host of `url` may be controlled by an attacker
regex = /^((www|beta).)?example.com/
if host.match(regex)
redirect_to url
end
end
end

3
ruby/ql/test/library-tests/regexp/regexp.expected
View File

@@ -6,11 +6,8 @@ groupNumber
| regexp.rb:46:2:46:6 | (foo) | 1 |
| regexp.rb:47:4:47:8 | (o\|b) | 1 |
| regexp.rb:48:2:48:9 | (a\|b\|cd) | 1 |
| regexp.rb:49:2:49:7 | (?::+) | 1 |
| regexp.rb:52:2:52:11 | (?<id>\\w+) | 1 |
| regexp.rb:53:2:53:12 | (?'foo'fo+) | 1 |
| regexp.rb:56:2:56:5 | (a+) | 1 |
| regexp.rb:57:2:57:11 | (?<qux>q+) | 1 |
term
| regexp.rb:5:2:5:4 | abc | RegExpConstant,RegExpNormalChar |
| regexp.rb:8:2:8:2 | a | RegExpConstant,RegExpNormalChar |

28
ruby/ql/test/query-tests/security/cwe-020/IncompleteHostnameRegExp/IncompleteHostnameRegExp.expected Normal file
View File

@@ -0,0 +1,28 @@
| hosttest.rb:1:18:1:41 | (www\|beta).example.com\\/ | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | hosttest.rb:1:17:1:42 | /(www\|beta).example.com\\// | here |
| hosttest.rb:2:33:2:55 | (www\|beta).example.com/ | This regular expression has an unescaped '.' before 'example.com/', so it might match more hosts than expected. | hosttest.rb:2:32:2:56 | "(www\|beta).example.com/" | here |
| hosttest.rb:3:29:3:51 | (www\|beta).example.com/ | This regular expression has an unescaped '.' before 'example.com/', so it might match more hosts than expected. | hosttest.rb:3:28:3:52 | "(www\|beta).example.com/" | here |
| tst-IncompleteHostnameRegExp.rb:3:3:3:28 | ^http:\\/\\/test.example.com | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.rb:3:2:3:29 | /^http:\\/\\/test.example.com/ | here |
| tst-IncompleteHostnameRegExp.rb:5:3:5:28 | ^http:\\/\\/test.example.net | This regular expression has an unescaped '.' before 'example.net', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.rb:5:2:5:29 | /^http:\\/\\/test.example.net/ | here |
| tst-IncompleteHostnameRegExp.rb:6:3:6:42 | ^http:\\/\\/test.(example-a\|example-b).com | This regular expression has an unescaped '.' before '(example-a\|example-b).com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.rb:6:2:6:43 | /^http:\\/\\/test.(example-a\|exa.../ | here |
| tst-IncompleteHostnameRegExp.rb:7:3:7:30 | ^http:\\/\\/(.+).example.com\\/ | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.rb:7:2:7:31 | /^http:\\/\\/(.+).example.com\\// | here |
| tst-IncompleteHostnameRegExp.rb:7:3:7:30 | ^http:\\/\\/(.+).example.com\\/ | This regular expression has an unrestricted wildcard '.+' which may cause 'example.com' to be matched anywhere in the URL, outside the hostname. | tst-IncompleteHostnameRegExp.rb:7:2:7:31 | /^http:\\/\\/(.+).example.com\\// | here |
| tst-IncompleteHostnameRegExp.rb:9:3:9:39 | ^http:\\/\\/(?:.+)\\.test\\.example.com\\/ | This regular expression has an unrestricted wildcard '.+' which may cause 'example.com' to be matched anywhere in the URL, outside the hostname. | tst-IncompleteHostnameRegExp.rb:9:2:9:40 | /^http:\\/\\/(?:.+)\\.test\\.examp.../ | here |
| tst-IncompleteHostnameRegExp.rb:10:3:10:36 | ^http:\\/\\/test.example.com\\/(?:.*) | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.rb:10:2:10:37 | /^http:\\/\\/test.example.com\\/(.../ | here |
| tst-IncompleteHostnameRegExp.rb:11:14:11:37 | ^http://test.example.com | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.rb:11:13:11:38 | "^http://test.example.com" | here |
| tst-IncompleteHostnameRegExp.rb:12:15:12:38 | ^http://test.example.com | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.rb:12:14:12:39 | "^http://test.example.com" | here |
| tst-IncompleteHostnameRegExp.rb:15:23:15:46 | ^http://test.example.com | This string, which is used as a regular expression $@, has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.rb:15:13:15:50 | call to id | here |
| tst-IncompleteHostnameRegExp.rb:17:14:17:30 | test.example.com$ | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.rb:17:13:17:31 | `test.example.com$` | here |
| tst-IncompleteHostnameRegExp.rb:19:14:19:30 | ^test.example.com | This string, which is used as a regular expression $@, has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.rb:20:13:20:26 | "#{...}$" | here |
| tst-IncompleteHostnameRegExp.rb:20:14:20:31 | ^test.example.com$ | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.rb:20:13:20:26 | "#{...}$" | here |
| tst-IncompleteHostnameRegExp.rb:37:3:37:53 | ^(https?:)?\\/\\/((service\|www).)?example.com(?=$\|\\/) | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.rb:37:2:37:54 | /^(https?:)?\\/\\/((service\|www).../ | here |
| tst-IncompleteHostnameRegExp.rb:38:3:38:43 | ^(http\|https):\\/\\/www.example.com\\/p\\/f\\/ | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.rb:38:2:38:44 | /^(http\|https):\\/\\/www.example.../ | here |
| tst-IncompleteHostnameRegExp.rb:39:5:39:30 | http:\\/\\/sub.example.com\\/ | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.rb:39:2:39:33 | /^(http:\\/\\/sub.example.com\\/)/ | here |
| tst-IncompleteHostnameRegExp.rb:40:3:40:29 | ^https?:\\/\\/api.example.com | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.rb:40:2:40:30 | /^https?:\\/\\/api.example.com/ | here |
| tst-IncompleteHostnameRegExp.rb:41:42:41:68 | ^https?://.+\\.example\\.com/ | This string, which is used as a regular expression $@, has an unrestricted wildcard '.+' which may cause 'example\\.com/' to be matched anywhere in the URL, outside the hostname. | tst-IncompleteHostnameRegExp.rb:41:13:41:71 | ... + ... | here |
| tst-IncompleteHostnameRegExp.rb:43:3:43:32 | ^https:\\/\\/[a-z]*.example.com$ | This regular expression has an unescaped '.' before 'example.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.rb:43:2:43:33 | /^https:\\/\\/[a-z]*.example.com$/ | here |
| tst-IncompleteHostnameRegExp.rb:44:40:44:53 | .+.example.net | This regular expression has an unescaped '.' before 'example.net', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.rb:44:17:44:109 | "^protos?://(localhost\|.+.exam..." | here |
| tst-IncompleteHostnameRegExp.rb:44:55:44:70 | .+.example-a.com | This regular expression has an unescaped '.' before 'example-a.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.rb:44:17:44:109 | "^protos?://(localhost\|.+.exam..." | here |
| tst-IncompleteHostnameRegExp.rb:44:72:44:87 | .+.example-b.com | This regular expression has an unescaped '.' before 'example-b.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.rb:44:17:44:109 | "^protos?://(localhost\|.+.exam..." | here |
| tst-IncompleteHostnameRegExp.rb:48:42:48:67 | ^https?://.+.example\\.com/ | This string, which is used as a regular expression $@, has an unescaped '.' before 'example\\.com/', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.rb:48:13:48:69 | ... + ... | here |
| tst-IncompleteHostnameRegExp.rb:48:42:48:67 | ^https?://.+.example\\.com/ | This string, which is used as a regular expression $@, has an unrestricted wildcard '.+' which may cause 'example\\.com/' to be matched anywhere in the URL, outside the hostname. | tst-IncompleteHostnameRegExp.rb:48:13:48:69 | ... + ... | here |
| tst-IncompleteHostnameRegExp.rb:59:5:59:20 | foo.example\\.com | This regular expression has an unescaped '.' before 'example\\.com', so it might match more hosts than expected. | tst-IncompleteHostnameRegExp.rb:59:2:59:32 | /^(foo.example\\.com\|whatever)$/ | here |

1
ruby/ql/test/query-tests/security/cwe-020/IncompleteHostnameRegExp/IncompleteHostnameRegExp.qlref Normal file
View File

@@ -0,0 +1 @@
queries/security/cwe-020/IncompleteHostnameRegExp.ql

22
ruby/ql/test/query-tests/security/cwe-020/IncompleteHostnameRegExp/hosttest.rb Normal file
View File

@@ -0,0 +1,22 @@
UNSAFE_REGEX1 = /(www|beta).example.com\//
UNSAFE_REGEX2 = Regexp.compile("(www|beta).example.com/")
UNSAFE_REGEX3 = Regexp.new("(www|beta).example.com/")
SAFE_REGEX = /(www|beta)\.example\.com\//
def unsafe
target = params[:target]
if UNSAFE_REGEX1.match(target)
redirect_to target
end
end
def safe
target = params[:target]
if SAFE_REGEX.match(target)
redirect_to target
end
end
# FP reported in https://github.com/github/codeql/issues/3712
# This does not define a regex (but could be used by other code to do so)
escaped = Regexp::escape("https://www.humblebundle.com/home/library")

67
ruby/ql/test/query-tests/security/cwe-020/IncompleteHostnameRegExp/tst-IncompleteHostnameRegExp.rb Normal file
View File

@@ -0,0 +1,67 @@
def foo
/^http:\/\/example.com/; # OK
/^http:\/\/test.example.com/; # NOT OK
/^http:\/\/test\.example.com/; # OK
/^http:\/\/test.example.net/; # NOT OK
/^http:\/\/test.(example-a|example-b).com/; # NOT OK
/^http:\/\/(.+).example.com\//; # NOT OK
/^http:\/\/(\.+)\.example.com/; # OK
/^http:\/\/(?:.+)\.test\.example.com\//; # NOT OK
/^http:\/\/test.example.com\/(?:.*)/; # OK
Regexp.new("^http://test.example.com"); # NOT OK
if (s.match("^http://test.example.com")); end # NOT OK
Regexp.new(id(id(id("^http://test.example.com")))); # NOT OK
Regexp.new(`test.example.com$`); # NOT OK
hostname = '^test.example.com'; # NOT OK
Regexp.new("#{hostname}$");
domain = { hostname: 'test.example.com$' }; # NOT OK
Regexp.new(domain[:hostname]);
convert1({ hostname: 'test.example.com$' }); # NOT OK
domains = [ { hostname: 'test.example.com$' } ]; # NOT OK
domains.map{ |d| convert2(d) };
/^(.+\.(?:example-a|example-b)\.com)\//; # NOT OK
/^(https?:)?\/\/((service|www).)?example.com(?=$|\/)/; # NOT OK
/^(http|https):\/\/www.example.com\/p\/f\//; # NOT OK
/^(http:\/\/sub.example.com\/)/i; # NOT OK
/^https?:\/\/api.example.com/; # NOT OK
Regexp.new('^http://localhost:8000|' + "^https?://.+\\.example\\.com/"); # NOT OK
Regexp.new("^http[s]?:\/\/?sub1\\.sub2\\.example\\.com\/f\/(.+)"); # NOT OK
/^https:\/\/[a-z]*.example.com$/; # NOT OK
Regexp.compile('^protos?://(localhost|.+.example.net|.+.example-a.com|.+.example-b.com|.+.example.internal)'); # NOT OK
/^(example.dev|example.com)/; # OK
Regexp.new('^http://localhost:8000|' + "^https?://.+.example\\.com/"); # NOT OK
primary = 'example.com$';
Regexp.new('test.' + primary); # NOT OK, but not detected
Regexp.new('test.' + 'example.com$'); # NOT OK
Regexp.new('^http://test\.example.com'); # NOT OK
/^http:\/\/(..|...)\.example\.com\/index\.html/; # OK, wildcards are intentional
/^http:\/\/.\.example\.com\/index\.html/; # OK, the wildcard is intentional
/^(foo.example\.com|whatever)$/; # kinda OK - one disjunction doesn't even look like a hostname
end
def id(e); return e; end
def convert1(domain)
return Regexp.new(domain[:hostname]);
end
def convert2(domain)
return Regexp.new(domain[:hostname]);
end

2
ruby/ql/test/query-tests/security/cwe-116/BadTagFilter.expected
View File

@@ -11,4 +11,4 @@
| test.rb:15:6:15:39 | <script[^>]*?>[\\s\\S]*?<\\/script.*> | This regular expression does not match script end tags like </script\\t\\n bar>. |
| test.rb:17:6:17:40 | <script\\b[^>]*>([\\s\\S]*?)<\\/script> | This regular expression does not match script end tags like </script >. |
| test.rb:18:6:18:48 | <(?:!--([\\S\|\\s]*?)-->)\|([^\\/\\s>]+)[\\S\\s]*?> | Comments ending with --> are matched differently from comments ending with --!>. The first is matched with capture group 1 and comments ending with --!> are matched with capture group 2. |
| test.rb:19:6:19:147 | <(?:(?:\\/([^>]+)>)\|(?:!--([\\S\|\\s]*?)-->)\|(?:([^\\/\\s>]+)((?:\\s+[\\w\\-:.]+(?:\\s*=\\s*?(?:(?:"[^"]*")\|(?:'[^']*')\|[^\\s"'\\/>]+))?)*)[\\S\\s]*?(\\/?)>)) | Comments ending with --> are matched differently from comments ending with --!>. The first is matched with capture group 2 and comments ending with --!> are matched with capture group 1, 3, 4, 5. |
| test.rb:19:6:19:147 | <(?:(?:\\/([^>]+)>)\|(?:!--([\\S\|\\s]*?)-->)\|(?:([^\\/\\s>]+)((?:\\s+[\\w\\-:.]+(?:\\s*=\\s*?(?:(?:"[^"]*")\|(?:'[^']*')\|[^\\s"'\\/>]+))?)*)[\\S\\s]*?(\\/?)>)) | Comments ending with --> are matched differently from comments ending with --!>. The first is matched with capture group 2 and comments ending with --!> are matched with capture group 3, 4. |