Merge pull request #12271 from github/tausbn/ql-fix-json-extraction

QL: Fix JSON extraction

ql/extractor/Cargo.toml

@@ -14,7 +14,7 @@ tree-sitter-ql = { git = "https://github.com/tree-sitter/tree-sitter-ql.git", re
 tree-sitter-ql-dbscheme = { git = "https://github.com/erik-krogh/tree-sitter-ql-dbscheme.git", rev = "63e1344353f63931e88bfbc2faa2e78e1421b213"}
 tree-sitter-ql-yaml = {git = "https://github.com/erik-krogh/tree-sitter-ql.git", rev = "cf704bf3671e1ae148e173464fb65a4d2bbf5f99"}
 tree-sitter-blame = {path = "../buramu/tree-sitter-blame"}
-tree-sitter-json = {git = "https://github.com/tausbn/tree-sitter-json.git", rev = "471ceac44d127e609afa349cf0a59370791fe8b3"}
+tree-sitter-json = {git = "https://github.com/tausbn/tree-sitter-json.git", rev = "ea1f655604c32c2f76aad2abed2498a56d81f3a9"}
 clap = "2.33"
 tracing = "0.1"
 tracing-subscriber = { version = "0.3.16", features = ["env-filter"] }

ql/extractor/src/main.rs

@@ -108,6 +108,9 @@ fn main() -> std::io::Result<()> {
                 && !line.ends_with(".dbscheme")
                 && !line.ends_with("qlpack.yml")
                 && !line.ends_with(".blame")
+                && !line.ends_with(".json")
+                && !line.ends_with(".jsonl")
+                && !line.ends_with(".jsonc")
             {
                 return Ok(());
             }

ql/generator/Cargo.toml

@@ -15,4 +15,4 @@ tree-sitter-ql = { git = "https://github.com/tree-sitter/tree-sitter-ql.git", re
 tree-sitter-ql-dbscheme = { git = "https://github.com/erik-krogh/tree-sitter-ql-dbscheme.git", rev = "63e1344353f63931e88bfbc2faa2e78e1421b213"}
 tree-sitter-ql-yaml = {git = "https://github.com/erik-krogh/tree-sitter-ql.git", rev = "cf704bf3671e1ae148e173464fb65a4d2bbf5f99"}
 tree-sitter-blame = {path = "../buramu/tree-sitter-blame"}
-tree-sitter-json = { git = "https://github.com/tausbn/tree-sitter-json.git", rev = "471ceac44d127e609afa349cf0a59370791fe8b3"}
+tree-sitter-json = { git = "https://github.com/tausbn/tree-sitter-json.git", rev = "ea1f655604c32c2f76aad2abed2498a56d81f3a9"}

ql/ql/src/codeql_ql/StructuredLogs.qll

@@ -0,0 +1,138 @@
+private import ql
+private import codeql_ql.ast.internal.TreeSitter
+
+class Object extends JSON::Object {
+  JSON::Value getValue(string key) {
+    exists(JSON::Pair p |
+      p = this.getChild(_) and p.getKey().(JSON::String).getChild().getValue() = key
+    |
+      result = p.getValue()
+    )
+  }
+
+  string getString(string key) { result = this.getValue(key).(JSON::String).getChild().getValue() }
+
+  int getNumber(string key) { result = this.getValue(key).(JSON::Number).getValue().toInt() }
+
+  Array getArray(string key) { result = this.getValue(key) }
+
+  Object getObject(string key) { result = this.getValue(key) }
+
+  string getType() { result = this.getString("type") }
+
+  int getEventId() { result = this.getNumber("event_id") }
+
+  string getTime() { result = this.getString("time") }
+}
+
+class Array extends JSON::Array {
+  Object getObject(int i) { result = this.getChild(i) }
+
+  string getString(int i) { result = this.getChild(i).(JSON::String).getChild().getValue() }
+
+  int getNumber(int i) { result = this.getChild(i).(JSON::Number).getValue().toInt() }
+
+  Array getArray(int i) { result = this.getChild(i) }
+}
+
+abstract class LogEntry extends Object { }
+
+class LogHeader extends LogEntry {
+  LogHeader() { this.getType() = "LOG_HEADER" }
+
+  string getCodeQLVersion() { result = this.getString("codeqlVersion") }
+
+  string getLogVersion() { result = this.getString("logVersion") }
+}
+
+class QueryStarted extends LogEntry {
+  QueryStarted() { this.getType() = "QUERY_STARTED" }
+
+  string getQueryName() { result = this.getString("queryName") }
+
+  int getStage(int i) { result = this.getArray("stage").getNumber(i) }
+}
+
+class PredicateStarted extends LogEntry {
+  PredicateStarted() { this.getType() = "PREDICATE_STARTED" }
+
+  string getPredicateName() { result = this.getString("predicateName") }
+
+  string getPosition() { result = this.getString("position") }
+
+  string getPredicateType() { result = this.getString("predicateType") }
+
+  int getQueryCausingWork() { result = this.getNumber("queryCausingWork") }
+
+  string getRAHash() { result = this.getString("raHash") }
+
+  Object getRA() { result = this.getValue("ra") }
+
+  string getDependency(string key) { result = this.getObject("dependencies").getString(key) }
+}
+
+class PipelineStarted extends LogEntry {
+  PipelineStarted() { this.getType() = "PIPELINE_STARTED" }
+
+  int getPredicateStartEvent() { result = this.getNumber("predicateStartEvent") }
+
+  string getRAReference() { result = this.getString("raReference") }
+}
+
+class PipelineCompleted extends LogEntry {
+  PipelineCompleted() { this.getType() = "PIPELINE_COMPLETED" }
+
+  int getStartEvent() { result = this.getNumber("startEvent") }
+
+  string getRAReference() { result = this.getString("raReference") }
+
+  int getCount(int i) { result = this.getArray("counts").getNumber(i) }
+
+  int getDuplicationPercentage(int i) {
+    result = this.getArray("duplicationPercentages").getNumber(i)
+  }
+
+  int getResultSize() { result = this.getNumber("resultSize") }
+}
+
+class PredicateCompleted extends LogEntry {
+  PredicateCompleted() { this.getType() = "PREDICATE_COMPLETED" }
+
+  int getStartEvent() { result = this.getNumber("startEvent") }
+
+  int getResultSize() { result = this.getNumber("resultSize") }
+}
+
+class QueryCompleted extends LogEntry {
+  QueryCompleted() { this.getType() = "QUERY_COMPLETED" }
+
+  int getStartEvent() { result = this.getNumber("startEvent") }
+
+  string getTerminationType() { result = this.getString("terminationType") }
+}
+
+class LogFooter extends LogEntry {
+  LogFooter() { this.getType() = "LOG_FOOTER" }
+}
+
+class CacheLookup extends LogEntry {
+  CacheLookup() { this.getType() = "CACHE_LOOKUP" }
+
+  int getRelationSize() { result = this.getNumber("relationSize") }
+}
+
+class SentinelEmpty extends LogEntry {
+  SentinelEmpty() { this.getType() = "SENTINEL_EMPTY" }
+}
+
+// Stuff to test whether we've covered all event types
+private File logFile() { result = any(LogHeader h).getLocation().getFile() }
+
+private Object missing() {
+  result =
+    any(Object o |
+      o.getLocation().getFile() = logFile() and
+      not o instanceof LogEntry and
+      not exists(o.getParent().getParent()) // don't count nested objects
+    )
+}

ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll

@@ -1930,12 +1930,6 @@ module JSON {
     final override AstNode getAFieldOrChild() { json_document_child(this, _, result) }
   }
 
-  /** A class representing `escape_sequence` tokens. */
-  class EscapeSequence extends @json_token_escape_sequence, Token {
-    /** Gets the name of the primary QL class for this element. */
-    final override string getAPrimaryQlClass() { result = "EscapeSequence" }
-  }
-
   /** A class representing `false` tokens. */
   class False extends @json_token_false, Token {
     /** Gets the name of the primary QL class for this element. */
@@ -1995,16 +1989,10 @@ module JSON {
     final override AstNode getAFieldOrChild() { json_string_child(this, result) }
   }
 
-  /** A class representing `string_content` nodes. */
-  class StringContent extends @json_string_content, AstNode {
+  /** A class representing `string_content` tokens. */
+  class StringContent extends @json_token_string_content, Token {
     /** Gets the name of the primary QL class for this element. */
     final override string getAPrimaryQlClass() { result = "StringContent" }
-
-    /** Gets the `i`th child of this node. */
-    final EscapeSequence getChild(int i) { json_string_content_child(this, i, result) }
-
-    /** Gets a field or child node of this node. */
-    final override AstNode getAFieldOrChild() { json_string_content_child(this, _, result) }
   }
 
   /** A class representing `true` tokens. */

ql/ql/src/ql.dbscheme

@@ -1292,24 +1292,13 @@ json_pair_def(
 
 json_string_child(
   unique int json_string__: @json_string__ ref,
-  unique int child: @json_string_content ref
+  unique int child: @json_token_string_content ref
 );
 
 json_string_def(
   unique int id: @json_string__
 );
 
-#keyset[json_string_content, index]
-json_string_content_child(
-  int json_string_content: @json_string_content ref,
-  int index: int ref,
-  unique int child: @json_token_escape_sequence ref
-);
-
-json_string_content_def(
-  unique int id: @json_string_content
-);
-
 @json_value = @json_array | @json_object | @json_string__ | @json_token_false | @json_token_null | @json_token_number | @json_token_true
 
 json_tokeninfo(
@@ -1321,15 +1310,15 @@ json_tokeninfo(
 case @json_token.kind of
   0 = @json_reserved_word
 | 1 = @json_token_comment
-| 2 = @json_token_escape_sequence
-| 3 = @json_token_false
-| 4 = @json_token_null
-| 5 = @json_token_number
+| 2 = @json_token_false
+| 3 = @json_token_null
+| 4 = @json_token_number
+| 5 = @json_token_string_content
 | 6 = @json_token_true
 ;
 
 
-@json_ast_node = @json_array | @json_document | @json_object | @json_pair | @json_string__ | @json_string_content | @json_token
+@json_ast_node = @json_array | @json_document | @json_object | @json_pair | @json_string__ | @json_token
 
 @json_ast_node_parent = @file | @json_ast_node