From 167574d82f392ad2bb7d199fb43c135cbf3a2350 Mon Sep 17 00:00:00 2001
From: Arthur Baars <aibaars@github.com>
Date: Tue, 16 Feb 2021 14:14:12 +0100
Subject: [PATCH 1/3] AST: HereDoc

---
 ql/src/codeql_ruby/ast/Literal.qll            | 46 +++++++++++++++-
 ql/src/codeql_ruby/ast/internal/AST.qll       |  6 ---
 ql/src/codeql_ruby/ast/internal/Literal.qll   | 52 ++++++++++++++++++-
 .../ast/literals/literals.expected            | 32 ++++++++++++
 .../library-tests/ast/literals/literals.rb    | 31 +++++++++++
 .../controlflow/graph/Cfg.expected            | 16 +++---
 6 files changed, 166 insertions(+), 17 deletions(-)

diff --git a/ql/src/codeql_ruby/ast/Literal.qll b/ql/src/codeql_ruby/ast/Literal.qll
index 30c9d0f51c9..75d038bc203 100644
--- a/ql/src/codeql_ruby/ast/Literal.qll
+++ b/ql/src/codeql_ruby/ast/Literal.qll
@@ -143,7 +143,7 @@ class StringComponent extends AstNode {
  * "foo#{ bar() } baz"
  * ```
  */
-class StringTextComponent extends StringComponent, @token_string_content {
+class StringTextComponent extends StringComponent, StringTextComponent::StringContentToken {
   final override StringTextComponent::Range range;
 
   final override string getAPrimaryQlClass() { result = "StringTextComponent" }
@@ -304,6 +304,50 @@ class CharacterLiteral extends Literal, @token_character {
   final override string getAPrimaryQlClass() { result = "CharacterLiteral" }
 }
 
+/**
+ * A "here document". For example:
+ * ```rb
+ * query = <<SQL
+ * SELECT * FROM person
+ * WHERE age > 21
+ * ```
+ */
+class HereDoc extends StringlikeLiteral {
+  final override HereDoc::Range range;
+
+  final override string getAPrimaryQlClass() { result = "HereDoc" }
+
+  /**
+   * Holds if this here document is executed in a subshell.
+   * ```rb
+   * <<`COMMAND`
+   * echo "Hello world!"
+   * COMMAND
+   * ```
+   */
+  final predicate isSubShell() { getQuoteStyle() = "`" }
+
+  /**
+   * Gets the quotation mark (`"`, `'` or `` ` ``) that surrounds the here document identifier, if any.
+   * ```rb
+   * <<"IDENTIFIER"
+   * <<'IDENTIFIER'
+   * <<`IDENTIFIER`
+   * ```
+   */
+  final string getQuoteStyle() { result = range.getQuoteStyle() }
+
+  /**
+   * Gets the indentation modifier (`-` or `~`) of the here document identifier, if any.
+   * ```rb
+   * <<~IDENTIFIER
+   * <<-IDENTIFIER
+   * <<IDENTIFIER
+   * ```
+   */
+  final string getIndentationModifier() { result = range.getIndentationModifier() }
+}
+
 /**
  * An array literal.
  *
diff --git a/ql/src/codeql_ruby/ast/internal/AST.qll b/ql/src/codeql_ruby/ast/internal/AST.qll
index 406622d85e6..b09355148a1 100644
--- a/ql/src/codeql_ruby/ast/internal/AST.qll
+++ b/ql/src/codeql_ruby/ast/internal/AST.qll
@@ -30,12 +30,6 @@ module AstNode {
       this instanceof Generated::RestAssignment
       or
       this instanceof Generated::Superclass
-      or
-      this instanceof Generated::HeredocBody
-      or
-      this instanceof Generated::HeredocBeginning
-      or
-      this instanceof Generated::HeredocEnd
     }
 
     override string toString() { result = "AstNode" }
diff --git a/ql/src/codeql_ruby/ast/internal/Literal.qll b/ql/src/codeql_ruby/ast/internal/Literal.qll
index b8ef438b70b..bd2243b93c5 100644
--- a/ql/src/codeql_ruby/ast/internal/Literal.qll
+++ b/ql/src/codeql_ruby/ast/internal/Literal.qll
@@ -94,8 +94,10 @@ module StringComponent {
 }
 
 module StringTextComponent {
-  class Range extends StringComponent::Range, @token_string_content {
-    final override Generated::StringContent generated;
+  class StringContentToken = @token_string_content or @token_heredoc_content;
+
+  class Range extends StringComponent::Range, StringContentToken {
+    final override Generated::Token generated;
 
     final override string toString() { result = generated.getValue() }
 
@@ -287,6 +289,52 @@ module CharacterLiteral {
   }
 }
 
+module HereDoc {
+  private Generated::HeredocBody heredoc(Generated::HeredocBeginning start) {
+    exists(int i, File f |
+      start =
+        rank[i](Generated::HeredocBeginning b |
+          f = b.getLocation().getFile()
+        |
+          b order by b.getLocation().getStartLine(), b.getLocation().getStartColumn()
+        ) and
+      result =
+        rank[i](Generated::HeredocBody b |
+          f = b.getLocation().getFile()
+        |
+          b order by b.getLocation().getStartLine(), b.getLocation().getStartColumn()
+        )
+    )
+  }
+
+  class Range extends StringlikeLiteral::Range, @token_heredoc_beginning {
+    final override Generated::HeredocBeginning generated;
+    private Generated::HeredocBody body;
+
+    Range() { body = heredoc(this) }
+
+    final override StringComponent::Range getComponent(int n) { result = body.getChild(n) }
+
+    final string getQuoteStyle() {
+      exists(string s |
+        s = generated.getValue() and
+        s.charAt(s.length() - 1) = result and
+        result = ["'", "`", "\""]
+      )
+    }
+
+    final string getIndentationModifier() {
+      exists(string s |
+        s = generated.getValue() and
+        s.charAt(2) = result and
+        result = ["-", "~"]
+      )
+    }
+
+    final override string toString() { result = generated.getValue() }
+  }
+}
+
 module ArrayLiteral {
   abstract class Range extends Literal::Range {
     final override string getValueText() { none() }
diff --git a/ql/test/library-tests/ast/literals/literals.expected b/ql/test/library-tests/ast/literals/literals.expected
index ef964f60b5a..f68a7025b79 100644
--- a/ql/test/library-tests/ast/literals/literals.expected
+++ b/ql/test/library-tests/ast/literals/literals.expected
@@ -195,6 +195,13 @@ allLiterals
 | literals.rb:145:1:145:34 | "abcdefghijklmnopqrstuvwxyzabcdef" | StringLiteral | abcdefghijklmnopqrstuvwxyzabcdef |
 | literals.rb:146:1:146:35 | "foobarfoobarfoobarfoobarfooba..." | StringLiteral | foobarfoobarfoobarfoobarfoobarfoo |
 | literals.rb:147:1:147:40 | "foobar\\\\foobar\\\\foobar\\\\fooba..." | StringLiteral | foobar\\\\foobar\\\\foobar\\\\foobar\\\\foobar |
+| literals.rb:150:9:150:13 | <<SQL | HereDoc | \nselect * from table\n |
+| literals.rb:150:16:150:20 | <<SQL | HereDoc | <none> |
+| literals.rb:157:11:157:16 | <<-BLA | HereDoc | \nsome text\\nand some more\n   |
+| literals.rb:162:9:162:19 | <<~SQUIGGLY | HereDoc | \n   indented stuff\n |
+| literals.rb:166:9:166:15 | <<"DOC" | HereDoc | <none> |
+| literals.rb:171:9:171:15 | <<'DOC' | HereDoc | <none> |
+| literals.rb:175:10:175:19 | <<`SCRIPT` | HereDoc | \n cat file.txt\n |
 stringlikeLiterals
 | literals.rb:46:1:46:2 | "" |  |
 | literals.rb:47:1:47:2 | "" |  |
@@ -285,6 +292,13 @@ stringlikeLiterals
 | literals.rb:145:1:145:34 | "abcdefghijklmnopqrstuvwxyzabcdef" | abcdefghijklmnopqrstuvwxyzabcdef |
 | literals.rb:146:1:146:35 | "foobarfoobarfoobarfoobarfooba..." | foobarfoobarfoobarfoobarfoobarfoo |
 | literals.rb:147:1:147:40 | "foobar\\\\foobar\\\\foobar\\\\fooba..." | foobar\\\\foobar\\\\foobar\\\\foobar\\\\foobar |
+| literals.rb:150:9:150:13 | <<SQL | \nselect * from table\n |
+| literals.rb:150:16:150:20 | <<SQL | <none> |
+| literals.rb:157:11:157:16 | <<-BLA | \nsome text\\nand some more\n   |
+| literals.rb:162:9:162:19 | <<~SQUIGGLY | \n   indented stuff\n |
+| literals.rb:166:9:166:15 | <<"DOC" | <none> |
+| literals.rb:171:9:171:15 | <<'DOC' | <none> |
+| literals.rb:175:10:175:19 | <<`SCRIPT` | \n cat file.txt\n |
 stringLiterals
 | literals.rb:46:1:46:2 | "" |  |
 | literals.rb:47:1:47:2 | "" |  |
@@ -492,6 +506,21 @@ stringComponents
 | literals.rb:147:1:147:40 | "foobar\\\\foobar\\\\foobar\\\\fooba..." | StringLiteral | 6 | literals.rb:147:26:147:31 | foobar | StringTextComponent |
 | literals.rb:147:1:147:40 | "foobar\\\\foobar\\\\foobar\\\\fooba..." | StringLiteral | 7 | literals.rb:147:32:147:33 | \\\\ | StringEscapeSequenceComponent |
 | literals.rb:147:1:147:40 | "foobar\\\\foobar\\\\foobar\\\\fooba..." | StringLiteral | 8 | literals.rb:147:34:147:39 | foobar | StringTextComponent |
+| literals.rb:150:9:150:13 | <<SQL | HereDoc | 0 | literals.rb:150:22:151:20 | \nselect * from table\n | StringTextComponent |
+| literals.rb:150:16:150:20 | <<SQL | HereDoc | 0 | literals.rb:152:4:153:13 | \nwhere name =  | StringTextComponent |
+| literals.rb:150:16:150:20 | <<SQL | HereDoc | 1 | literals.rb:153:14:153:22 | #{...} | StringInterpolationComponent |
+| literals.rb:150:16:150:20 | <<SQL | HereDoc | 2 | literals.rb:153:23:153:23 | \n | StringTextComponent |
+| literals.rb:157:11:157:16 | <<-BLA | HereDoc | 0 | literals.rb:157:17:158:9 | \nsome text | StringTextComponent |
+| literals.rb:157:11:157:16 | <<-BLA | HereDoc | 1 | literals.rb:158:10:158:11 | \\n | StringEscapeSequenceComponent |
+| literals.rb:157:11:157:16 | <<-BLA | HereDoc | 2 | literals.rb:158:12:159:2 | and some more\n   | StringTextComponent |
+| literals.rb:162:9:162:19 | <<~SQUIGGLY | HereDoc | 0 | literals.rb:162:20:163:18 | \n   indented stuff\n | StringTextComponent |
+| literals.rb:166:9:166:15 | <<"DOC" | HereDoc | 0 | literals.rb:166:16:167:11 | \n text with  | StringTextComponent |
+| literals.rb:166:9:166:15 | <<"DOC" | HereDoc | 1 | literals.rb:167:12:167:29 | #{...} | StringInterpolationComponent |
+| literals.rb:166:9:166:15 | <<"DOC" | HereDoc | 2 | literals.rb:167:30:167:32 |  !\n | StringTextComponent |
+| literals.rb:171:9:171:15 | <<'DOC' | HereDoc | 0 | literals.rb:171:16:172:14 | \n text without  | StringTextComponent |
+| literals.rb:171:9:171:15 | <<'DOC' | HereDoc | 1 | literals.rb:172:15:172:32 | #{...} | StringInterpolationComponent |
+| literals.rb:171:9:171:15 | <<'DOC' | HereDoc | 2 | literals.rb:172:33:172:35 |  !\n | StringTextComponent |
+| literals.rb:175:10:175:19 | <<`SCRIPT` | HereDoc | 0 | literals.rb:175:20:176:14 | \n cat file.txt\n | StringTextComponent |
 stringInterpolations
 | literals.rb:58:10:58:19 | #{...} | 0 | literals.rb:58:13:58:17 | ... + ... | AddExpr |
 | literals.rb:59:12:59:21 | #{...} | 0 | literals.rb:59:15:59:19 | ... + ... | AddExpr |
@@ -505,6 +534,9 @@ stringInterpolations
 | literals.rb:128:10:128:19 | #{...} | 0 | literals.rb:128:13:128:17 | ... - ... | SubExpr |
 | literals.rb:135:5:135:14 | #{...} | 0 | literals.rb:135:8:135:12 | ... + ... | AddExpr |
 | literals.rb:141:7:141:16 | #{...} | 0 | literals.rb:141:10:141:14 | ... + ... | AddExpr |
+| literals.rb:153:14:153:22 | #{...} | 0 | literals.rb:153:17:153:20 | call to name | Call |
+| literals.rb:167:12:167:29 | #{...} | 0 | literals.rb:167:15:167:27 | call to interpolation | Call |
+| literals.rb:172:15:172:32 | #{...} | 0 | literals.rb:172:18:172:30 | call to interpolation | Call |
 concatenatedStrings
 | literals.rb:62:1:62:17 | "..." "..." | foobarbaz | 0 | literals.rb:62:1:62:5 | "foo" |
 | literals.rb:62:1:62:17 | "..." "..." | foobarbaz | 1 | literals.rb:62:7:62:11 | "bar" |
diff --git a/ql/test/library-tests/ast/literals/literals.rb b/ql/test/library-tests/ast/literals/literals.rb
index 2202223c9bf..02cb635809a 100644
--- a/ql/test/library-tests/ast/literals/literals.rb
+++ b/ql/test/library-tests/ast/literals/literals.rb
@@ -145,3 +145,34 @@ TRUE
 'abcdefghijklmnopqrstuvwxyzabcdef'  # 32 chars, should not be truncated
 'foobarfoobarfoobarfoobarfoobarfoo' # 33 chars, should be truncated
 "foobar\\foobar\\foobar\\foobar\\foobar" # several short components, but long enough overall to be truncated
+
+# here documents
+run_sql(<<SQL, <<SQL)
+select * from table
+SQL
+where name = #{ name }
+SQL
+
+def m 
+  query = <<-BLA
+some text\nand some more
+  BLA
+end
+
+query = <<~SQUIGGLY
+   indented stuff
+SQUIGGLY
+
+query = <<"DOC"
+ text with #{ interpolation } !
+DOC
+
+# TODO: the parser currently does not handle single quoted heredocs correctly
+query = <<'DOC'
+ text without #{ interpolation } !
+DOC
+
+output = <<`SCRIPT`
+ cat file.txt
+SCRIPT
+
diff --git a/ql/test/library-tests/controlflow/graph/Cfg.expected b/ql/test/library-tests/controlflow/graph/Cfg.expected
index 67c03b0ff51..259244f2028 100644
--- a/ql/test/library-tests/controlflow/graph/Cfg.expected
+++ b/ql/test/library-tests/controlflow/graph/Cfg.expected
@@ -1331,14 +1331,14 @@ cfg.rb:
 #  108| puts
 #-----|  -> <<SQL
 
-#  108| ()
+#  108| (<<SQL)
 #-----|  -> call to puts
 
 #  108| <<SQL
 #-----|  -> call to table
 
-#  108| HeredocBody
-#-----|  -> ()
+#  108| (no string representation)
+#-----|  -> (<<SQL)
 
 #  109| #{...}
 #-----|  -> call to type
@@ -1347,7 +1347,7 @@ cfg.rb:
 #-----|  -> #{...}
 
 #  110| #{...}
-#-----|  -> HeredocBody
+#-----|  -> (no string representation)
 
 #  110| call to type
 #-----|  -> #{...}
@@ -2154,15 +2154,15 @@ heredoc.rb:
 #-----|  -> <<A
 
 #    2| <<A
-#-----|  -> HeredocBody
+#-----|  -> (no string representation)
 
 #    2| <<A
-#-----|  -> HeredocBody
+#-----|  -> (no string representation)
 
-#    2| HeredocBody
+#    2| (no string representation)
 #-----|  -> <<A
 
-#    4| HeredocBody
+#    4| (no string representation)
 #-----|  -> call to puts
 
 ifs.rb:

From e1047fad2ccddf637f4e7453e2ec3d4e9e357ee8 Mon Sep 17 00:00:00 2001
From: Arthur Baars <aibaars@github.com>
Date: Tue, 16 Feb 2021 21:10:17 +0100
Subject: [PATCH 2/3] CFG: remove intermediate HeredocBody nodes

---
 .../controlflow/internal/ControlFlowGraphImpl.qll   |  7 ++-----
 .../library-tests/controlflow/graph/Cfg.expected    | 13 ++-----------
 2 files changed, 4 insertions(+), 16 deletions(-)

diff --git a/ql/src/codeql_ruby/controlflow/internal/ControlFlowGraphImpl.qll b/ql/src/codeql_ruby/controlflow/internal/ControlFlowGraphImpl.qll
index 01e535ffbab..eb95602c468 100644
--- a/ql/src/codeql_ruby/controlflow/internal/ControlFlowGraphImpl.qll
+++ b/ql/src/codeql_ruby/controlflow/internal/ControlFlowGraphImpl.qll
@@ -316,7 +316,7 @@ private class LeftToRightPostOrderNodes =
   @argument_list or @array or @bare_string or @bare_symbol or @binary or @block_argument or
       @break or @call or @chained_string or @delimited_symbol or @destructured_left_assignment or
       @destructured_parameter or @element_reference or @exception_variable or @hash or
-      @hash_splat_argument or @heredoc_body or @interpolation or @left_assignment_list or @next or
+      @hash_splat_argument or @interpolation or @left_assignment_list or @next or
       @operator_assignment or @pair or @parenthesized_statements or @range or @redo or @regex or
       @rest_assignment or @retry or @return or @right_assignment_list or @scope_resolution or
       @token_simple_symbol or @splat_argument or @string__ or @string_array or @subshell or
@@ -684,10 +684,7 @@ module Trees {
   }
 
   private class HeredocBeginningTree extends StandardPreOrderTree, HeredocBeginning {
-    final override ControlFlowTree getChildNode(int i) {
-      i = 0 and
-      result = heredoc(this)
-    }
+    final override ControlFlowTree getChildNode(int i) { result = heredoc(this).getChild(i) }
   }
 
   private class IdentifierTree extends LeafTree, Identifier { }
diff --git a/ql/test/library-tests/controlflow/graph/Cfg.expected b/ql/test/library-tests/controlflow/graph/Cfg.expected
index 259244f2028..8e3167bc71d 100644
--- a/ql/test/library-tests/controlflow/graph/Cfg.expected
+++ b/ql/test/library-tests/controlflow/graph/Cfg.expected
@@ -1337,9 +1337,6 @@ cfg.rb:
 #  108| <<SQL
 #-----|  -> call to table
 
-#  108| (no string representation)
-#-----|  -> (<<SQL)
-
 #  109| #{...}
 #-----|  -> call to type
 
@@ -1347,7 +1344,7 @@ cfg.rb:
 #-----|  -> #{...}
 
 #  110| #{...}
-#-----|  -> (no string representation)
+#-----|  -> (<<SQL)
 
 #  110| call to type
 #-----|  -> #{...}
@@ -2154,15 +2151,9 @@ heredoc.rb:
 #-----|  -> <<A
 
 #    2| <<A
-#-----|  -> (no string representation)
-
-#    2| <<A
-#-----|  -> (no string representation)
-
-#    2| (no string representation)
 #-----|  -> <<A
 
-#    4| (no string representation)
+#    2| <<A
 #-----|  -> call to puts
 
 ifs.rb:

From cabe6df82056ed44daea8213baf1cc37774e1d76 Mon Sep 17 00:00:00 2001
From: Arthur Baars <aibaars@github.com>
Date: Wed, 17 Feb 2021 15:58:13 +0100
Subject: [PATCH 3/3] Add missing heredoc end token

---
 ql/src/codeql_ruby/ast/Literal.qll | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ql/src/codeql_ruby/ast/Literal.qll b/ql/src/codeql_ruby/ast/Literal.qll
index 75d038bc203..00aa5359acb 100644
--- a/ql/src/codeql_ruby/ast/Literal.qll
+++ b/ql/src/codeql_ruby/ast/Literal.qll
@@ -310,6 +310,7 @@ class CharacterLiteral extends Literal, @token_character {
  * query = <<SQL
  * SELECT * FROM person
  * WHERE age > 21
+ * SQL
  * ```
  */
 class HereDoc extends StringlikeLiteral {