diff --git a/javascript/extractor/src/com/semmle/js/extractor/ASTExtractor.java b/javascript/extractor/src/com/semmle/js/extractor/ASTExtractor.java index 686fa4c6803..7898c4b8e34 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/ASTExtractor.java +++ b/javascript/extractor/src/com/semmle/js/extractor/ASTExtractor.java @@ -6,6 +6,7 @@ import java.util.Collections; import java.util.List; import java.util.Set; import java.util.Stack; +import java.util.regex.Matcher; import com.semmle.js.ast.AClass; import com.semmle.js.ast.AFunction; @@ -570,6 +571,16 @@ public class ASTExtractor { regexpExtractor.extract(source.substring(1, source.lastIndexOf('/')), offsets, nd, false); } else if (nd.isStringLiteral() && !c.isInsideType() && nd.getRaw().length() < 1000) { regexpExtractor.extract(valueString, makeStringLiteralOffsets(nd.getRaw()), nd, true); + + // Scan the string for template tags, if we're in a context where such tags are relevant. + if (scopeManager.isInTemplateFile()) { + Matcher m = TemplateEngines.TEMPLATE_TAGS.matcher(nd.getRaw()); + int offset = nd.getLoc().getStart().getOffset(); + while (m.find()) { + Label locationLbl = TemplateEngines.makeLocation(lexicalExtractor.getTextualExtractor(), offset + m.start(), offset + m.end()); + trapwriter.addTuple("expr_contains_template_tag_location", key, locationLbl); + } + } } return key; } @@ -2225,7 +2236,8 @@ public class ASTExtractor { @Override public Label visit(GeneratedCodeExpr nd, Context c) { Label key = super.visit(nd, c); - trapwriter.addTuple("generated_code_expr_info", key, nd.getOpeningDelimiter(), nd.getClosingDelimiter(), nd.getBody()); + Label templateLbl = TemplateEngines.makeLocation(lexicalExtractor.getTextualExtractor(), nd.getLoc().getStart().getOffset(), nd.getLoc().getEnd().getOffset()); + trapwriter.addTuple("expr_contains_template_tag_location", key, templateLbl); return key; } } diff --git a/javascript/extractor/src/com/semmle/js/extractor/HTMLExtractor.java b/javascript/extractor/src/com/semmle/js/extractor/HTMLExtractor.java index f740742ee28..526e539d171 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/HTMLExtractor.java +++ b/javascript/extractor/src/com/semmle/js/extractor/HTMLExtractor.java @@ -39,7 +39,7 @@ public class HTMLExtractor implements IExtractor { this.textualExtractor = textualExtractor; this.scopeManager = - new ScopeManager(textualExtractor.getTrapwriter(), config.getEcmaVersion()); + new ScopeManager(textualExtractor.getTrapwriter(), config.getEcmaVersion(), true); } /* @@ -382,21 +382,6 @@ public class HTMLExtractor implements IExtractor { writer.addTuple("toplevel_parent_xml_node", topLevelLabel, htmlNodeLabel); } - private static final String MUSTACHE_TAG_DOUBLE = "\\{\\{(?!\\{)(.*?)\\}\\}"; // {{ x }} - private static final String MUSTACHE_TAG_TRIPLE = "\\{\\{\\{(.*?)\\}\\}\\}"; // {{{ x }}} - private static final String MUSTACHE_TAG_PERCENT = "\\{%(?!>)(.*?)%\\}"; // {% x %} - private static final String EJS_TAG = "<%(?![%<>}])[-=]?(.*?)[_-]?%>"; // <% x %> - - /** Pattern for a template tag whose contents should be parsed as an expression */ - private static final Pattern TEMPLATE_EXPR_OPENING_TAG = - Pattern.compile("^(?:\\{\\{\\{?|<%[-=])"); // {{, {{{, <%=, <%- - - private static final Pattern TEMPLATE_TAGS = - Pattern.compile( - StringUtil.glue( - "|", MUSTACHE_TAG_DOUBLE, MUSTACHE_TAG_TRIPLE, MUSTACHE_TAG_PERCENT, EJS_TAG), - Pattern.DOTALL); - private void extractTemplateTags( TextualExtractor textualExtractor, ScopeManager scopeManager, @@ -407,9 +392,8 @@ public class HTMLExtractor implements IExtractor { if (start >= end) return; if (isEmbedded) return; // Do not extract template tags for HTML snippets embedded in a JS file - LocationManager locationManager = textualExtractor.getLocationManager(); TrapWriter trapwriter = textualExtractor.getTrapwriter(); - Matcher m = TEMPLATE_TAGS.matcher(textualExtractor.getSource()).region(start, end); + Matcher m = TemplateEngines.TEMPLATE_TAGS.matcher(textualExtractor.getSource()).region(start, end); while (m.find()) { int startOffset = m.start(); int endOffset = m.end(); @@ -424,15 +408,12 @@ public class HTMLExtractor implements IExtractor { String rawText = m.group(); trapwriter.addTuple("template_placeholder_tag_info", lbl, parentLabel.get(), rawText); - // Emit location - Position startPos = textualExtractor.getSourceMap().getStart(startOffset); - Position endPos = textualExtractor.getSourceMap().getEnd(endOffset - 1); - int endColumn = endPos.getColumn() - 1; // Convert to inclusive end position (still 1-based) - locationManager.emitFileLocation( - lbl, startPos.getLine(), startPos.getColumn(), endPos.getLine(), endColumn); + // Emit location entity + Label locationLbl = TemplateEngines.makeLocation(textualExtractor, startOffset, endOffset); + trapwriter.addTuple("hasLocation", lbl, locationLbl); // Parse the contents as a template expression, if the delimiter expects an expression. - Matcher delimMatcher = TEMPLATE_EXPR_OPENING_TAG.matcher(rawText); + Matcher delimMatcher = TemplateEngines.TEMPLATE_EXPR_OPENING_TAG.matcher(rawText); if (delimMatcher.find()) { // The body of the template tag is stored in the first capture group of each pattern int bodyGroup = getNonNullCaptureGroup(m); diff --git a/javascript/extractor/src/com/semmle/js/extractor/ScopeManager.java b/javascript/extractor/src/com/semmle/js/extractor/ScopeManager.java index b738e875415..81bdc9e6f92 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/ScopeManager.java +++ b/javascript/extractor/src/com/semmle/js/extractor/ScopeManager.java @@ -103,12 +103,22 @@ public class ScopeManager { private final ECMAVersion ecmaVersion; private final Set implicitGlobals = new LinkedHashSet(); private Scope implicitVariableScope; + private final boolean isInTemplateScope; - public ScopeManager(TrapWriter trapWriter, ECMAVersion ecmaVersion) { + public ScopeManager(TrapWriter trapWriter, ECMAVersion ecmaVersion, boolean isInTemplateScope) { this.trapWriter = trapWriter; this.toplevelScope = enterScope(ScopeKind.GLOBAL, trapWriter.globalID("global_scope"), null); this.ecmaVersion = ecmaVersion; this.implicitVariableScope = toplevelScope; + this.isInTemplateScope = isInTemplateScope; + } + + /** + * Returns true the current scope is potentially in a template file, and may contain + * relevant template tags. + */ + public boolean isInTemplateFile() { + return isInTemplateScope; } /** diff --git a/javascript/extractor/src/com/semmle/js/extractor/ScriptExtractor.java b/javascript/extractor/src/com/semmle/js/extractor/ScriptExtractor.java index 344c144648e..08d29d98a9b 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/ScriptExtractor.java +++ b/javascript/extractor/src/com/semmle/js/extractor/ScriptExtractor.java @@ -77,7 +77,7 @@ public class ScriptExtractor implements IExtractor { } ScopeManager scopeManager = - new ScopeManager(textualExtractor.getTrapwriter(), config.getEcmaVersion()); + new ScopeManager(textualExtractor.getTrapwriter(), config.getEcmaVersion(), false); Label toplevelLabel = null; LoCInfo loc; try { diff --git a/javascript/extractor/src/com/semmle/js/extractor/TemplateEngines.java b/javascript/extractor/src/com/semmle/js/extractor/TemplateEngines.java new file mode 100644 index 00000000000..895e1f84b7b --- /dev/null +++ b/javascript/extractor/src/com/semmle/js/extractor/TemplateEngines.java @@ -0,0 +1,37 @@ +package com.semmle.js.extractor; + +import java.util.regex.Pattern; + +import com.semmle.util.data.StringUtil; +import com.semmle.util.locations.Position; +import com.semmle.util.trap.TrapWriter.Label; + +public class TemplateEngines { + private static final String MUSTACHE_TAG_TRIPLE = "\\{\\{\\{(.*?)\\}\\}\\}"; // {{{ x }}} + private static final String MUSTACHE_TAG_DOUBLE = "\\{\\{(?!\\{)(.*?)\\}\\}"; // {{ x }}} + private static final String MUSTACHE_TAG_PERCENT = "\\{%(?!>)(.*?)%\\}"; // {% x %} + private static final String EJS_TAG = "<%(?![%<>}])[-=]?(.*?)[_-]?%>"; // <% x %> + + /** Pattern for a template tag whose contents should be parsed as an expression */ + public static final Pattern TEMPLATE_EXPR_OPENING_TAG = + Pattern.compile("^(?:\\{\\{\\{?|<%[-=])"); // {{, {{{, <%=, <%- + + /** + * Pattern matching a template tag from a supported template engine. + */ + public static final Pattern TEMPLATE_TAGS = + Pattern.compile( + StringUtil.glue( + "|", TemplateEngines.MUSTACHE_TAG_DOUBLE, MUSTACHE_TAG_TRIPLE, MUSTACHE_TAG_PERCENT, EJS_TAG), + Pattern.DOTALL); + + /** + * Returns the location label for a template tag at the given offsets. + */ + public static Label makeLocation(TextualExtractor extractor, int startOffset, int endOffset) { + Position startPos = extractor.getSourceMap().getStart(startOffset); + Position endPos = extractor.getSourceMap().getEnd(endOffset - 1); + int endColumn = endPos.getColumn() - 1; // Convert to inclusive end position (still 1-based) + return extractor.getLocationManager().emitLocationsDefault(startPos.getLine(), startPos.getColumn(), endPos.getLine(), endColumn); + } +} diff --git a/javascript/extractor/src/com/semmle/js/extractor/TypeScriptExtractor.java b/javascript/extractor/src/com/semmle/js/extractor/TypeScriptExtractor.java index 219dc46a9bc..629abccb422 100644 --- a/javascript/extractor/src/com/semmle/js/extractor/TypeScriptExtractor.java +++ b/javascript/extractor/src/com/semmle/js/extractor/TypeScriptExtractor.java @@ -23,7 +23,7 @@ public class TypeScriptExtractor implements IExtractor { File sourceFile = textualExtractor.getExtractedFile(); Result res = state.getTypeScriptParser().parse(sourceFile, source, textualExtractor.getMetrics()); ScopeManager scopeManager = - new ScopeManager(textualExtractor.getTrapwriter(), ECMAVersion.ECMA2017); + new ScopeManager(textualExtractor.getTrapwriter(), ECMAVersion.ECMA2017, false); try { FileSnippet snippet = state.getSnippets().get(sourceFile.toPath()); SourceType sourceType = snippet != null ? snippet.getSourceType() : jsExtractor.establishSourceType(source, false); diff --git a/javascript/ql/src/semmle/javascript/Expr.qll b/javascript/ql/src/semmle/javascript/Expr.qll index b9a77e8b08f..4ce07a30627 100644 --- a/javascript/ql/src/semmle/javascript/Expr.qll +++ b/javascript/ql/src/semmle/javascript/Expr.qll @@ -2893,18 +2893,9 @@ class ImportMetaExpr extends @import_meta_expr, Expr { * ``` */ class GeneratedCodeExpr extends @generated_code_expr, Expr { - /** Gets the opening delimiter, such as `{{` or `{{{`. */ - string getOpeningDelimiter() { generated_code_expr_info(this, result, _, _) } - - /** Gets the closing delimiter, such as `}}` or `}}}`. */ - string getClosingDelimiter() { generated_code_expr_info(this, _, result, _) } - - /** Gets the text between the delimiters, including any surrounding whitespace, such as the `x` in `{{x}}`. */ - string getBody() { generated_code_expr_info(this, _, _, result) } - /** Gets the placeholder tag that was parsed as an expression. */ Templating::TemplatePlaceholderTag getPlaceholderTag() { - result.getLocation() = this.getLocation() + this = result.getEnclosingExpr() } override string getAPrimaryQlClass() { result = "GeneratedCodeExpr" } diff --git a/javascript/ql/src/semmle/javascript/frameworks/Templating.qll b/javascript/ql/src/semmle/javascript/frameworks/Templating.qll index d18646aacf7..ff92f60e23f 100644 --- a/javascript/ql/src/semmle/javascript/frameworks/Templating.qll +++ b/javascript/ql/src/semmle/javascript/frameworks/Templating.qll @@ -114,6 +114,13 @@ module Templating { * that is, without being enclosed in a string literal or similar. */ predicate isInPlainCodeContext() { this = any(GeneratedCodeExpr e).getPlaceholderTag() } + + /** + * Gets the innermost JavaScript expression containing this template tag, if any. + */ + Expr getEnclosingExpr() { + expr_contains_template_tag_location(result, getLocation()) + } } /** diff --git a/javascript/ql/src/semmlecode.javascript.dbscheme b/javascript/ql/src/semmlecode.javascript.dbscheme index bfd181ef8af..9ca3692b65f 100644 --- a/javascript/ql/src/semmlecode.javascript.dbscheme +++ b/javascript/ql/src/semmlecode.javascript.dbscheme @@ -417,11 +417,9 @@ case @expr.kind of @e4x_xml_attribute_selector = @e4x_xml_static_attribute_selector | @e4x_xml_dynamic_attribute_selector; @e4x_xml_qualident = @e4x_xml_static_qualident | @e4x_xml_dynamic_qualident; -generated_code_expr_info( - unique int expr: @generated_code_expr ref, - varchar(900) openingDelimiter: string ref, - varchar(900) closingDelimiter: string ref, - varchar(900) body: string ref +expr_contains_template_tag_location( + int expr: @expr ref, + int location: @location ref ); @template_placeholder_tag_parent = @xmlelement | @xmlattribute | @file;