From 2bbf1e37fc23c4ab33563cdd96e0d8181f30fb37 Mon Sep 17 00:00:00 2001
From: Taus <tausbn@github.com>
Date: Sun, 30 May 2021 17:38:26 +0000
Subject: [PATCH 01/10] QL: Add support for dbscheme and qlpack.yml

Currently the the YAML "parser" fails for some legacy files, but the
actual files appear to be parsed correctly.
---
 Cargo.lock                                   |   2 +-
 extractor/Cargo.toml                         |   2 +-
 generator/Cargo.toml                         |   2 +-
 ql/src/codeql_ql/ast/Ast.qll                 |   2 +-
 ql/src/codeql_ql/ast/internal/TreeSitter.qll | 295 ++++++++++++++++++-
 ql/src/ql.dbscheme                           | 219 ++++++++++++--
 tools/autobuild.cmd                          |   2 +
 tools/autobuild.sh                           |   2 +
 8 files changed, 496 insertions(+), 30 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index a361f14d9cd..47f52dc3da5 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -589,7 +589,7 @@ dependencies = [
 [[package]]
 name = "tree-sitter-ql"
 version = "0.19.0"
-source = "git+https://github.com/tausbn/tree-sitter-ql.git?rev=ef46f6c4c076cc06bb206686794cec26b5c544a7#ef46f6c4c076cc06bb206686794cec26b5c544a7"
+source = "git+https://github.com/tausbn/tree-sitter-ql.git?rev=e08901378e2fd5c33f1e88bbb59716fd36f094f7#e08901378e2fd5c33f1e88bbb59716fd36f094f7"
 dependencies = [
  "cc",
  "tree-sitter",
diff --git a/extractor/Cargo.toml b/extractor/Cargo.toml
index bd5e0e0e4bb..e19913ed4a8 100644
--- a/extractor/Cargo.toml
+++ b/extractor/Cargo.toml
@@ -10,7 +10,7 @@ edition = "2018"
 flate2 = "1.0"
 node-types = { path = "../node-types" }
 tree-sitter = "0.19"
-tree-sitter-ql = { git = "https://github.com/tausbn/tree-sitter-ql.git", rev = "ef46f6c4c076cc06bb206686794cec26b5c544a7" }
+tree-sitter-ql = { git = "https://github.com/tausbn/tree-sitter-ql.git", rev = "e08901378e2fd5c33f1e88bbb59716fd36f094f7" }
 clap = "2.33"
 tracing = "0.1"
 tracing-subscriber = { version = "0.2", features = ["env-filter"] }
diff --git a/generator/Cargo.toml b/generator/Cargo.toml
index e996c498e59..ce609782065 100644
--- a/generator/Cargo.toml
+++ b/generator/Cargo.toml
@@ -10,4 +10,4 @@ edition = "2018"
 node-types = { path = "../node-types" }
 tracing = "0.1"
 tracing-subscriber = { version = "0.2", features = ["env-filter"] }
-tree-sitter-ql = { git = "https://github.com/tausbn/tree-sitter-ql.git", rev = "ef46f6c4c076cc06bb206686794cec26b5c544a7" }
+tree-sitter-ql = { git = "https://github.com/tausbn/tree-sitter-ql.git", rev = "e08901378e2fd5c33f1e88bbb59716fd36f094f7" }
diff --git a/ql/src/codeql_ql/ast/Ast.qll b/ql/src/codeql_ql/ast/Ast.qll
index 418e4102d72..e93f1c257d5 100644
--- a/ql/src/codeql_ql/ast/Ast.qll
+++ b/ql/src/codeql_ql/ast/Ast.qll
@@ -77,7 +77,7 @@ class TopLevel extends TTopLevel, AstNode {
    * Gets a member from contained in this top-level module.
    * Includes private members.
    */
-  ModuleMember getAMember() { toGenerated(result) = file.getChild(_).getChild(_) }
+  ModuleMember getAMember() { toGenerated(result) = file.getChild(_).(Generated::ModuleMember).getChild(_) }
 
   override ModuleMember getAChild(string pred) {
     pred = directMember("getAMember") and result = this.getAMember()
diff --git a/ql/src/codeql_ql/ast/internal/TreeSitter.qll b/ql/src/codeql_ql/ast/internal/TreeSitter.qll
index cb1f1be2dd2..031b7def05d 100644
--- a/ql/src/codeql_ql/ast/internal/TreeSitter.qll
+++ b/ql/src/codeql_ql/ast/internal/TreeSitter.qll
@@ -438,6 +438,273 @@ module Generated {
     override AstNode getAFieldOrChild() { datatype_branches_child(this, _, result) }
   }
 
+  /** A class representing `db_annotation` nodes. */
+  class DbAnnotation extends @db_annotation, AstNode {
+    /** Gets the name of the primary QL class for this element. */
+    override string getAPrimaryQlClass() { result = "DbAnnotation" }
+
+    /** Gets the location of this element. */
+    override Location getLocation() { db_annotation_def(this, result) }
+
+    /** Gets the node corresponding to the field `argsAnnotation`. */
+    DbArgsAnnotation getArgsAnnotation() { db_annotation_args_annotation(this, result) }
+
+    /** Gets the node corresponding to the field `simpleAnnotation`. */
+    AnnotName getSimpleAnnotation() { db_annotation_simple_annotation(this, result) }
+
+    /** Gets a field or child node of this node. */
+    override AstNode getAFieldOrChild() {
+      db_annotation_args_annotation(this, result) or db_annotation_simple_annotation(this, result)
+    }
+  }
+
+  /** A class representing `db_argsAnnotation` nodes. */
+  class DbArgsAnnotation extends @db_args_annotation, AstNode {
+    /** Gets the name of the primary QL class for this element. */
+    override string getAPrimaryQlClass() { result = "DbArgsAnnotation" }
+
+    /** Gets the location of this element. */
+    override Location getLocation() { db_args_annotation_def(this, _, result) }
+
+    /** Gets the node corresponding to the field `name`. */
+    AnnotName getName() { db_args_annotation_def(this, result, _) }
+
+    SimpleId getChild(int i) { db_args_annotation_child(this, i, result) }
+
+    /** Gets a field or child node of this node. */
+    override AstNode getAFieldOrChild() {
+      db_args_annotation_def(this, result, _) or db_args_annotation_child(this, _, result)
+    }
+  }
+
+  /** A class representing `db_boolean` tokens. */
+  class DbBoolean extends @token_db_boolean, Token {
+    /** Gets the name of the primary QL class for this element. */
+    override string getAPrimaryQlClass() { result = "DbBoolean" }
+  }
+
+  /** A class representing `db_branch` nodes. */
+  class DbBranch extends @db_branch, AstNode {
+    /** Gets the name of the primary QL class for this element. */
+    override string getAPrimaryQlClass() { result = "DbBranch" }
+
+    /** Gets the location of this element. */
+    override Location getLocation() { db_branch_def(this, result) }
+
+    /** Gets the node corresponding to the field `qldoc`. */
+    Qldoc getQldoc() { db_branch_qldoc(this, result) }
+
+    AstNode getChild(int i) { db_branch_child(this, i, result) }
+
+    /** Gets a field or child node of this node. */
+    override AstNode getAFieldOrChild() {
+      db_branch_qldoc(this, result) or db_branch_child(this, _, result)
+    }
+  }
+
+  /** A class representing `db_case` tokens. */
+  class DbCase extends @token_db_case, Token {
+    /** Gets the name of the primary QL class for this element. */
+    override string getAPrimaryQlClass() { result = "DbCase" }
+  }
+
+  /** A class representing `db_caseDecl` nodes. */
+  class DbCaseDecl extends @db_case_decl, AstNode {
+    /** Gets the name of the primary QL class for this element. */
+    override string getAPrimaryQlClass() { result = "DbCaseDecl" }
+
+    /** Gets the location of this element. */
+    override Location getLocation() { db_case_decl_def(this, _, _, result) }
+
+    /** Gets the node corresponding to the field `base`. */
+    Dbtype getBase() { db_case_decl_def(this, result, _, _) }
+
+    /** Gets the node corresponding to the field `discriminator`. */
+    SimpleId getDiscriminator() { db_case_decl_def(this, _, result, _) }
+
+    AstNode getChild(int i) { db_case_decl_child(this, i, result) }
+
+    /** Gets a field or child node of this node. */
+    override AstNode getAFieldOrChild() {
+      db_case_decl_def(this, result, _, _) or
+      db_case_decl_def(this, _, result, _) or
+      db_case_decl_child(this, _, result)
+    }
+  }
+
+  /** A class representing `db_colType` nodes. */
+  class DbColType extends @db_col_type, AstNode {
+    /** Gets the name of the primary QL class for this element. */
+    override string getAPrimaryQlClass() { result = "DbColType" }
+
+    /** Gets the location of this element. */
+    override Location getLocation() { db_col_type_def(this, _, result) }
+
+    AstNode getChild() { db_col_type_def(this, result, _) }
+
+    /** Gets a field or child node of this node. */
+    override AstNode getAFieldOrChild() { db_col_type_def(this, result, _) }
+  }
+
+  /** A class representing `db_column` nodes. */
+  class DbColumn extends @db_column, AstNode {
+    /** Gets the name of the primary QL class for this element. */
+    override string getAPrimaryQlClass() { result = "DbColumn" }
+
+    /** Gets the location of this element. */
+    override Location getLocation() { db_column_def(this, _, _, _, result) }
+
+    /** Gets the node corresponding to the field `colName`. */
+    SimpleId getColName() { db_column_def(this, result, _, _, _) }
+
+    /** Gets the node corresponding to the field `colType`. */
+    DbColType getColType() { db_column_def(this, _, result, _, _) }
+
+    /** Gets the node corresponding to the field `isRef`. */
+    DbRef getIsRef() { db_column_is_ref(this, result) }
+
+    /** Gets the node corresponding to the field `isUnique`. */
+    DbUnique getIsUnique() { db_column_is_unique(this, result) }
+
+    /** Gets the node corresponding to the field `qldoc`. */
+    Qldoc getQldoc() { db_column_qldoc(this, result) }
+
+    /** Gets the node corresponding to the field `reprType`. */
+    DbReprType getReprType() { db_column_def(this, _, _, result, _) }
+
+    /** Gets a field or child node of this node. */
+    override AstNode getAFieldOrChild() {
+      db_column_def(this, result, _, _, _) or
+      db_column_def(this, _, result, _, _) or
+      db_column_is_ref(this, result) or
+      db_column_is_unique(this, result) or
+      db_column_qldoc(this, result) or
+      db_column_def(this, _, _, result, _)
+    }
+  }
+
+  /** A class representing `db_date` tokens. */
+  class DbDate extends @token_db_date, Token {
+    /** Gets the name of the primary QL class for this element. */
+    override string getAPrimaryQlClass() { result = "DbDate" }
+  }
+
+  /** A class representing `db_entry` nodes. */
+  class DbEntry extends @db_entry, AstNode {
+    /** Gets the name of the primary QL class for this element. */
+    override string getAPrimaryQlClass() { result = "DbEntry" }
+
+    /** Gets the location of this element. */
+    override Location getLocation() { db_entry_def(this, _, result) }
+
+    AstNode getChild() { db_entry_def(this, result, _) }
+
+    /** Gets a field or child node of this node. */
+    override AstNode getAFieldOrChild() { db_entry_def(this, result, _) }
+  }
+
+  /** A class representing `db_float` tokens. */
+  class DbFloat extends @token_db_float, Token {
+    /** Gets the name of the primary QL class for this element. */
+    override string getAPrimaryQlClass() { result = "DbFloat" }
+  }
+
+  /** A class representing `db_int` tokens. */
+  class DbInt extends @token_db_int, Token {
+    /** Gets the name of the primary QL class for this element. */
+    override string getAPrimaryQlClass() { result = "DbInt" }
+  }
+
+  /** A class representing `db_ref` tokens. */
+  class DbRef extends @token_db_ref, Token {
+    /** Gets the name of the primary QL class for this element. */
+    override string getAPrimaryQlClass() { result = "DbRef" }
+  }
+
+  /** A class representing `db_reprType` nodes. */
+  class DbReprType extends @db_repr_type, AstNode {
+    /** Gets the name of the primary QL class for this element. */
+    override string getAPrimaryQlClass() { result = "DbReprType" }
+
+    /** Gets the location of this element. */
+    override Location getLocation() { db_repr_type_def(this, result) }
+
+    AstNode getChild(int i) { db_repr_type_child(this, i, result) }
+
+    /** Gets a field or child node of this node. */
+    override AstNode getAFieldOrChild() { db_repr_type_child(this, _, result) }
+  }
+
+  /** A class representing `db_string` tokens. */
+  class DbString extends @token_db_string, Token {
+    /** Gets the name of the primary QL class for this element. */
+    override string getAPrimaryQlClass() { result = "DbString" }
+  }
+
+  /** A class representing `db_table` nodes. */
+  class DbTable extends @db_table, AstNode {
+    /** Gets the name of the primary QL class for this element. */
+    override string getAPrimaryQlClass() { result = "DbTable" }
+
+    /** Gets the location of this element. */
+    override Location getLocation() { db_table_def(this, _, result) }
+
+    /** Gets the node corresponding to the field `tableName`. */
+    DbTableName getTableName() { db_table_def(this, result, _) }
+
+    AstNode getChild(int i) { db_table_child(this, i, result) }
+
+    /** Gets a field or child node of this node. */
+    override AstNode getAFieldOrChild() {
+      db_table_def(this, result, _) or db_table_child(this, _, result)
+    }
+  }
+
+  /** A class representing `db_tableName` nodes. */
+  class DbTableName extends @db_table_name, AstNode {
+    /** Gets the name of the primary QL class for this element. */
+    override string getAPrimaryQlClass() { result = "DbTableName" }
+
+    /** Gets the location of this element. */
+    override Location getLocation() { db_table_name_def(this, _, result) }
+
+    SimpleId getChild() { db_table_name_def(this, result, _) }
+
+    /** Gets a field or child node of this node. */
+    override AstNode getAFieldOrChild() { db_table_name_def(this, result, _) }
+  }
+
+  /** A class representing `db_unionDecl` nodes. */
+  class DbUnionDecl extends @db_union_decl, AstNode {
+    /** Gets the name of the primary QL class for this element. */
+    override string getAPrimaryQlClass() { result = "DbUnionDecl" }
+
+    /** Gets the location of this element. */
+    override Location getLocation() { db_union_decl_def(this, _, result) }
+
+    /** Gets the node corresponding to the field `base`. */
+    Dbtype getBase() { db_union_decl_def(this, result, _) }
+
+    Dbtype getChild(int i) { db_union_decl_child(this, i, result) }
+
+    /** Gets a field or child node of this node. */
+    override AstNode getAFieldOrChild() {
+      db_union_decl_def(this, result, _) or db_union_decl_child(this, _, result)
+    }
+  }
+
+  /** A class representing `db_unique` tokens. */
+  class DbUnique extends @token_db_unique, Token {
+    /** Gets the name of the primary QL class for this element. */
+    override string getAPrimaryQlClass() { result = "DbUnique" }
+  }
+
+  /** A class representing `db_varchar` tokens. */
+  class DbVarchar extends @token_db_varchar, Token {
+    /** Gets the name of the primary QL class for this element. */
+    override string getAPrimaryQlClass() { result = "DbVarchar" }
+  }
+
   /** A class representing `dbtype` tokens. */
   class Dbtype extends @token_dbtype, Token {
     /** Gets the name of the primary QL class for this element. */
@@ -995,7 +1262,7 @@ module Generated {
     /** Gets the location of this element. */
     override Location getLocation() { ql_def(this, result) }
 
-    ModuleMember getChild(int i) { ql_child(this, i, result) }
+    AstNode getChild(int i) { ql_child(this, i, result) }
 
     /** Gets a field or child node of this node. */
     override AstNode getAFieldOrChild() { ql_child(this, _, result) }
@@ -1345,4 +1612,30 @@ module Generated {
     /** Gets a field or child node of this node. */
     override AstNode getAFieldOrChild() { variable_def(this, result, _) }
   }
+
+  /** A class representing `yaml_entry` nodes. */
+  class YamlEntry extends @yaml_entry, AstNode {
+    /** Gets the name of the primary QL class for this element. */
+    override string getAPrimaryQlClass() { result = "YamlEntry" }
+
+    /** Gets the location of this element. */
+    override Location getLocation() { yaml_entry_def(this, _, _, result) }
+
+    /** Gets the node corresponding to the field `key`. */
+    SimpleId getKey() { yaml_entry_def(this, result, _, _) }
+
+    /** Gets the node corresponding to the field `value`. */
+    YamlValue getValue() { yaml_entry_def(this, _, result, _) }
+
+    /** Gets a field or child node of this node. */
+    override AstNode getAFieldOrChild() {
+      yaml_entry_def(this, result, _, _) or yaml_entry_def(this, _, result, _)
+    }
+  }
+
+  /** A class representing `yaml_value` tokens. */
+  class YamlValue extends @token_yaml_value, Token {
+    /** Gets the name of the primary QL class for this element. */
+    override string getAPrimaryQlClass() { result = "YamlValue" }
+  }
 }
diff --git a/ql/src/ql.dbscheme b/ql/src/ql.dbscheme
index a0e8d866cf6..1e6d6473b49 100644
--- a/ql/src/ql.dbscheme
+++ b/ql/src/ql.dbscheme
@@ -289,6 +289,156 @@ datatype_branches_def(
   int loc: @location ref
 );
 
+db_annotation_args_annotation(
+  unique int db_annotation: @db_annotation ref,
+  unique int argsAnnotation: @db_args_annotation ref
+);
+
+db_annotation_simple_annotation(
+  unique int db_annotation: @db_annotation ref,
+  unique int simpleAnnotation: @token_annot_name ref
+);
+
+db_annotation_def(
+  unique int id: @db_annotation,
+  int loc: @location ref
+);
+
+#keyset[db_args_annotation, index]
+db_args_annotation_child(
+  int db_args_annotation: @db_args_annotation ref,
+  int index: int ref,
+  unique int child: @token_simple_id ref
+);
+
+db_args_annotation_def(
+  unique int id: @db_args_annotation,
+  int name: @token_annot_name ref,
+  int loc: @location ref
+);
+
+db_branch_qldoc(
+  unique int db_branch: @db_branch ref,
+  unique int qldoc: @token_qldoc ref
+);
+
+@db_branch_child_type = @token_dbtype | @token_integer
+
+#keyset[db_branch, index]
+db_branch_child(
+  int db_branch: @db_branch ref,
+  int index: int ref,
+  unique int child: @db_branch_child_type ref
+);
+
+db_branch_def(
+  unique int id: @db_branch,
+  int loc: @location ref
+);
+
+@db_caseDecl_child_type = @db_branch | @token_db_case
+
+#keyset[db_case_decl, index]
+db_case_decl_child(
+  int db_case_decl: @db_case_decl ref,
+  int index: int ref,
+  unique int child: @db_caseDecl_child_type ref
+);
+
+db_case_decl_def(
+  unique int id: @db_case_decl,
+  int base: @token_dbtype ref,
+  int discriminator: @token_simple_id ref,
+  int loc: @location ref
+);
+
+@db_colType_child_type = @token_db_boolean | @token_db_date | @token_db_float | @token_db_int | @token_db_string | @token_dbtype
+
+db_col_type_def(
+  unique int id: @db_col_type,
+  int child: @db_colType_child_type ref,
+  int loc: @location ref
+);
+
+db_column_is_ref(
+  unique int db_column: @db_column ref,
+  unique int isRef: @token_db_ref ref
+);
+
+db_column_is_unique(
+  unique int db_column: @db_column ref,
+  unique int isUnique: @token_db_unique ref
+);
+
+db_column_qldoc(
+  unique int db_column: @db_column ref,
+  unique int qldoc: @token_qldoc ref
+);
+
+db_column_def(
+  unique int id: @db_column,
+  int col_name: @token_simple_id ref,
+  int col_type: @db_col_type ref,
+  int repr_type: @db_repr_type ref,
+  int loc: @location ref
+);
+
+@db_entry_child_type = @db_case_decl | @db_table | @db_union_decl | @token_qldoc
+
+db_entry_def(
+  unique int id: @db_entry,
+  int child: @db_entry_child_type ref,
+  int loc: @location ref
+);
+
+@db_reprType_child_type = @token_db_boolean | @token_db_date | @token_db_float | @token_db_int | @token_db_string | @token_db_varchar | @token_integer
+
+#keyset[db_repr_type, index]
+db_repr_type_child(
+  int db_repr_type: @db_repr_type ref,
+  int index: int ref,
+  unique int child: @db_reprType_child_type ref
+);
+
+db_repr_type_def(
+  unique int id: @db_repr_type,
+  int loc: @location ref
+);
+
+@db_table_child_type = @db_annotation | @db_column
+
+#keyset[db_table, index]
+db_table_child(
+  int db_table: @db_table ref,
+  int index: int ref,
+  unique int child: @db_table_child_type ref
+);
+
+db_table_def(
+  unique int id: @db_table,
+  int table_name: @db_table_name ref,
+  int loc: @location ref
+);
+
+db_table_name_def(
+  unique int id: @db_table_name,
+  int child: @token_simple_id ref,
+  int loc: @location ref
+);
+
+#keyset[db_union_decl, index]
+db_union_decl_child(
+  int db_union_decl: @db_union_decl ref,
+  int index: int ref,
+  unique int child: @token_dbtype ref
+);
+
+db_union_decl_def(
+  unique int id: @db_union_decl,
+  int base: @token_dbtype ref,
+  int loc: @location ref
+);
+
 @disjunction_left_type = @add_expr | @aggregate | @call_or_unqual_agg_expr | @comp_term | @conjunction | @disjunction | @expr_annotation | @if_term | @implication | @in_expr | @instance_of | @literal | @mul_expr | @negation | @par_expr | @prefix_cast | @qualified_expr | @quantified | @range | @set_literal | @special_call | @super_ref | @unary_expr | @variable
 
 @disjunction_right_type = @add_expr | @aggregate | @call_or_unqual_agg_expr | @comp_term | @conjunction | @disjunction | @expr_annotation | @if_term | @implication | @in_expr | @instance_of | @literal | @mul_expr | @negation | @par_expr | @prefix_cast | @qualified_expr | @quantified | @range | @set_literal | @special_call | @super_ref | @unary_expr | @variable
@@ -616,11 +766,13 @@ prefix_cast_def(
   int loc: @location ref
 );
 
+@ql_child_type = @db_entry | @module_member | @yaml_entry
+
 #keyset[ql, index]
 ql_child(
   int ql: @ql ref,
   int index: int ref,
-  unique int child: @module_member ref
+  unique int child: @ql_child_type ref
 );
 
 ql_def(
@@ -872,6 +1024,13 @@ variable_def(
   int loc: @location ref
 );
 
+yaml_entry_def(
+  unique int id: @yaml_entry,
+  int key__: @token_simple_id ref,
+  int value: @token_yaml_value ref,
+  int loc: @location ref
+);
+
 tokeninfo(
   unique int id: @token,
   int kind: int ref,
@@ -890,29 +1049,39 @@ case @token.kind of
 | 5 = @token_class_name
 | 6 = @token_closure
 | 7 = @token_compop
-| 8 = @token_dbtype
-| 9 = @token_direction
-| 10 = @token_empty
-| 11 = @token_false
-| 12 = @token_float
-| 13 = @token_integer
-| 14 = @token_line_comment
-| 15 = @token_literal_id
-| 16 = @token_mulop
-| 17 = @token_predicate
-| 18 = @token_predicate_name
-| 19 = @token_primitive_type
-| 20 = @token_qldoc
-| 21 = @token_quantifier
-| 22 = @token_result
-| 23 = @token_simple_id
-| 24 = @token_special_id
-| 25 = @token_string
-| 26 = @token_super
-| 27 = @token_this
-| 28 = @token_true
-| 29 = @token_underscore
-| 30 = @token_unop
+| 8 = @token_db_boolean
+| 9 = @token_db_case
+| 10 = @token_db_date
+| 11 = @token_db_float
+| 12 = @token_db_int
+| 13 = @token_db_ref
+| 14 = @token_db_string
+| 15 = @token_db_unique
+| 16 = @token_db_varchar
+| 17 = @token_dbtype
+| 18 = @token_direction
+| 19 = @token_empty
+| 20 = @token_false
+| 21 = @token_float
+| 22 = @token_integer
+| 23 = @token_line_comment
+| 24 = @token_literal_id
+| 25 = @token_mulop
+| 26 = @token_predicate
+| 27 = @token_predicate_name
+| 28 = @token_primitive_type
+| 29 = @token_qldoc
+| 30 = @token_quantifier
+| 31 = @token_result
+| 32 = @token_simple_id
+| 33 = @token_special_id
+| 34 = @token_string
+| 35 = @token_super
+| 36 = @token_this
+| 37 = @token_true
+| 38 = @token_underscore
+| 39 = @token_unop
+| 40 = @token_yaml_value
 ;
 
 
@@ -933,7 +1102,7 @@ case @diagnostic.severity of
 ;
 
 
-@ast_node = @add_expr | @aggregate | @annot_arg | @annotation | @arityless_predicate_expr | @as_expr | @as_exprs | @body | @bool | @call_body | @call_or_unqual_agg_expr | @charpred | @class_member | @classless_predicate | @comp_term | @conjunction | @dataclass | @datatype | @datatype_branch | @datatype_branches | @disjunction | @expr_aggregate_body | @expr_annotation | @field | @full_aggregate_body | @higher_order_term | @if_term | @implication | @import_directive | @import_module_expr | @in_expr | @instance_of | @literal | @member_predicate | @module | @module_alias_body | @module_expr | @module_member | @module_name | @mul_expr | @negation | @order_by | @order_bys | @par_expr | @predicate_alias_body | @predicate_expr | @prefix_cast | @ql | @qual_module_expr | @qualified_expr | @qualified_rhs | @quantified | @range | @select | @set_literal | @special_call | @super_ref | @token | @type_alias_body | @type_expr | @type_union_body | @unary_expr | @unqual_agg_body | @var_decl | @var_name | @variable
+@ast_node = @add_expr | @aggregate | @annot_arg | @annotation | @arityless_predicate_expr | @as_expr | @as_exprs | @body | @bool | @call_body | @call_or_unqual_agg_expr | @charpred | @class_member | @classless_predicate | @comp_term | @conjunction | @dataclass | @datatype | @datatype_branch | @datatype_branches | @db_annotation | @db_args_annotation | @db_branch | @db_case_decl | @db_col_type | @db_column | @db_entry | @db_repr_type | @db_table | @db_table_name | @db_union_decl | @disjunction | @expr_aggregate_body | @expr_annotation | @field | @full_aggregate_body | @higher_order_term | @if_term | @implication | @import_directive | @import_module_expr | @in_expr | @instance_of | @literal | @member_predicate | @module | @module_alias_body | @module_expr | @module_member | @module_name | @mul_expr | @negation | @order_by | @order_bys | @par_expr | @predicate_alias_body | @predicate_expr | @prefix_cast | @ql | @qual_module_expr | @qualified_expr | @qualified_rhs | @quantified | @range | @select | @set_literal | @special_call | @super_ref | @token | @type_alias_body | @type_expr | @type_union_body | @unary_expr | @unqual_agg_body | @var_decl | @var_name | @variable | @yaml_entry
 
 @ast_node_parent = @ast_node | @file
 
diff --git a/tools/autobuild.cmd b/tools/autobuild.cmd
index aa9dde24d58..b000600fea3 100644
--- a/tools/autobuild.cmd
+++ b/tools/autobuild.cmd
@@ -3,6 +3,8 @@
 type NUL && "%CODEQL_DIST%\codeql.exe" database index-files ^
     --include-extension=.ql ^
     --include-extension=.qll ^
+    --include-extension=.dbscheme ^
+    --include=**/qlpack.yml ^
     --size-limit=5m ^
     --language=ql ^
     "%CODEQL_EXTRACTOR_QL_WIP_DATABASE%"
diff --git a/tools/autobuild.sh b/tools/autobuild.sh
index ba30bd74b3b..ef682d0980b 100755
--- a/tools/autobuild.sh
+++ b/tools/autobuild.sh
@@ -5,6 +5,8 @@ set -eu
 exec "${CODEQL_DIST}/codeql" database index-files \
     --include-extension=.ql \
     --include-extension=.qll \
+    --include-extension=.dbscheme \
+    --include=**/qlpack.yml \
     --size-limit=5m \
     --language=ql \
     --working-dir=.\

From 676bc5ee2afd82364e4b529ff4650e353c12d220 Mon Sep 17 00:00:00 2001
From: Taus <tausbn@github.com>
Date: Sun, 30 May 2021 17:53:38 +0000
Subject: [PATCH 02/10] QL: Autoformat

---
 ql/src/codeql_ql/ast/Ast.qll | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/ql/src/codeql_ql/ast/Ast.qll b/ql/src/codeql_ql/ast/Ast.qll
index e93f1c257d5..2f0320d55c9 100644
--- a/ql/src/codeql_ql/ast/Ast.qll
+++ b/ql/src/codeql_ql/ast/Ast.qll
@@ -77,7 +77,9 @@ class TopLevel extends TTopLevel, AstNode {
    * Gets a member from contained in this top-level module.
    * Includes private members.
    */
-  ModuleMember getAMember() { toGenerated(result) = file.getChild(_).(Generated::ModuleMember).getChild(_) }
+  ModuleMember getAMember() {
+    toGenerated(result) = file.getChild(_).(Generated::ModuleMember).getChild(_)
+  }
 
   override ModuleMember getAChild(string pred) {
     pred = directMember("getAMember") and result = this.getAMember()

From 1aa7cbb918e33f4d77957ca6ac6dbf43b3b26ce7 Mon Sep 17 00:00:00 2001
From: Erik Krogh Kristensen <erik-krogh@github.com>
Date: Sun, 30 May 2021 21:30:11 +0000
Subject: [PATCH 03/10] QL: use qlpack existence to determine the root for
 imports

---
 ql/src/codeql_ql/ast/internal/Module.qll | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/ql/src/codeql_ql/ast/internal/Module.qll b/ql/src/codeql_ql/ast/internal/Module.qll
index 0df304ba6c8..1b9ac5b0d49 100644
--- a/ql/src/codeql_ql/ast/internal/Module.qll
+++ b/ql/src/codeql_ql/ast/internal/Module.qll
@@ -48,7 +48,11 @@ private class Folder_ extends ContainerOrModule, TFolder {
 
   Folder_() { this = TFolder(f) }
 
-  override ContainerOrModule getEnclosing() { result = TFolder(f.getParentContainer()) }
+  override ContainerOrModule getEnclosing() {
+    result = TFolder(f.getParentContainer()) and
+    // if this the the root, then we stop.
+    not exists(f.getFile("qlpack.yml"))
+  }
 
   override string getName() { result = f.getStem() }
 
@@ -63,6 +67,11 @@ private class Folder_ extends ContainerOrModule, TFolder {
     endline = 0 and
     endcolumn = 0
   }
+
+  /**
+   * Gets the folder that this IPA type represents.
+   */
+  Folder getFolder() { result = f }
 }
 
 // TODO: Use `AstNode::getParent` once it is total
@@ -108,6 +117,7 @@ private predicate resolveQualifiedName(Import imp, ContainerOrModule m, int i) {
       exists(Container c, Container parent |
         // should ideally look at `qlpack.yml` files
         parent = imp.getLocation().getFile().getParentContainer+() and
+        exists(parent.getFile("qlpack.yml")) and
         c.getParentContainer() = parent and
         q = m.getName()
       |
@@ -116,7 +126,17 @@ private predicate resolveQualifiedName(Import imp, ContainerOrModule m, int i) {
         m = TFolder(c)
       )
       or
-      definesModule(getEnclosingModule(imp).getEnclosing*(), q, m, _)
+      q = imp.getQualifiedName(i) and
+      exists(ContainerOrModule container | container = getEnclosingModule(imp).getEnclosing+() |
+        definesModule(container, q, m, _) and
+        (
+          exists(container.(Folder_).getFolder().getFile("qlpack.yml")) or
+          container.(Folder_).getFolder() = imp.getLocation().getFile().getParentContainer() or
+          not container instanceof Folder_
+        )
+      )
+      or
+      definesModule(getEnclosingModule(imp), q, m, _)
     )
     or
     exists(Folder_ mid |

From e47c4ff2ad7690efa98757a5fff521e2d624e6a7 Mon Sep 17 00:00:00 2001
From: Erik Krogh Kristensen <erik-krogh@github.com>
Date: Sun, 30 May 2021 21:36:01 +0000
Subject: [PATCH 04/10] QL: spaces in folder/file names are underscores in
 import names

---
 ql/src/codeql_ql/ast/internal/Module.qll | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ql/src/codeql_ql/ast/internal/Module.qll b/ql/src/codeql_ql/ast/internal/Module.qll
index 1b9ac5b0d49..210e7f6b352 100644
--- a/ql/src/codeql_ql/ast/internal/Module.qll
+++ b/ql/src/codeql_ql/ast/internal/Module.qll
@@ -28,7 +28,7 @@ private class File_ extends FileOrModule, TFile {
 
   override ContainerOrModule getEnclosing() { result = TFolder(f.getParentContainer()) }
 
-  override string getName() { result = f.getStem() }
+  override string getName() { result = f.getStem().replaceAll(" ", "_") }
 
   override string toString() { result = f.toString() }
 
@@ -54,7 +54,7 @@ private class Folder_ extends ContainerOrModule, TFolder {
     not exists(f.getFile("qlpack.yml"))
   }
 
-  override string getName() { result = f.getStem() }
+  override string getName() { result = f.getStem().replaceAll(" ", "_") }
 
   override string toString() { result = f.toString() }
 

From 5e89bf99a827bb42441279923ee9083d1df72180 Mon Sep 17 00:00:00 2001
From: Erik Krogh Kristensen <erik-krogh@github.com>
Date: Sun, 30 May 2021 21:42:15 +0000
Subject: [PATCH 05/10] QL: fix printAst performance

---
 ql/src/codeql_ql/printAstAst.qll | 50 ++++++++++++--------------------
 1 file changed, 18 insertions(+), 32 deletions(-)

diff --git a/ql/src/codeql_ql/printAstAst.qll b/ql/src/codeql_ql/printAstAst.qll
index dc2ae735dc2..42603fe3752 100644
--- a/ql/src/codeql_ql/printAstAst.qll
+++ b/ql/src/codeql_ql/printAstAst.qll
@@ -28,7 +28,6 @@ class PrintAstConfiguration extends string {
  * The ordering is location based and pretty arbitary.
  */
 AstNode getAstChild(PrintAstNode parent, int i) {
-  parent.shouldPrint() and
   result =
     rank[i](AstNode child, Location l |
       child.getParent() = parent and
@@ -44,32 +43,24 @@ AstNode getAstChild(PrintAstNode parent, int i) {
  * A node in the output tree.
  */
 class PrintAstNode extends AstNode {
-  string getProperty(string key) {
-    this.shouldPrint() and
-    (
-      key = "semmle.label" and
-      result = "[" + concat(this.getAPrimaryQlClass(), ", ") + "] " + this.toString()
-      or
-      key = "semmle.order" and
-      result =
-        any(int i |
-          this =
-            rank[i](AstNode p, Location l, File f |
-              l = p.getLocation() and
-              f = l.getFile()
-            |
-              p order by f.getBaseName(), f.getAbsolutePath(), l.getStartLine(), l.getStartColumn()
-            )
-        ).toString()
-    )
-  }
+  PrintAstNode() { shouldPrintNode(this) }
 
-  /**
-   * Holds if this node should be printed in the output. By default, all nodes
-   * are printed, but the query can override
-   * `PrintAstConfiguration.shouldPrintNode` to filter the output.
-   */
-  predicate shouldPrint() { shouldPrintNode(this) }
+  string getProperty(string key) {
+    key = "semmle.label" and
+    result = "[" + concat(this.getAPrimaryQlClass(), ", ") + "] " + this.toString()
+    or
+    key = "semmle.order" and
+    result =
+      any(int i |
+        this =
+          rank[i](AstNode p, Location l, File f |
+            l = p.getLocation() and
+            f = l.getFile()
+          |
+            p order by f.getBaseName(), f.getAbsolutePath(), l.getStartLine(), l.getStartColumn()
+          )
+      ).toString()
+  }
 
   /**
    * Gets the child node that is accessed using the predicate `edgeName`.
@@ -85,18 +76,13 @@ private predicate shouldPrintNode(AstNode n) {
  * Holds if `node` belongs to the output tree, and its property `key` has the
  * given `value`.
  */
-query predicate nodes(PrintAstNode node, string key, string value) {
-  node.shouldPrint() and
-  value = node.getProperty(key)
-}
+query predicate nodes(PrintAstNode node, string key, string value) { value = node.getProperty(key) }
 
 /**
  * Holds if `target` is a child of `source` in the AST, and property `key` of
  * the edge has the given `value`.
  */
 query predicate edges(PrintAstNode source, PrintAstNode target, string key, string value) {
-  source.shouldPrint() and
-  target.shouldPrint() and
   target = source.getChild(_) and
   (
     key = "semmle.label" and

From 7a4e4134dadb6036ef43e98b0299e397fb421a3f Mon Sep 17 00:00:00 2001
From: Erik Krogh Kristensen <erik-krogh@github.com>
Date: Mon, 31 May 2021 07:33:11 +0000
Subject: [PATCH 06/10] QL: more improvements to printAst performance

---
 ql/src/codeql_ql/printAstAst.qll       |  2 +-
 ql/src/codeql_ql/printAstGenerated.qll | 50 ++++++++++----------------
 2 files changed, 19 insertions(+), 33 deletions(-)

diff --git a/ql/src/codeql_ql/printAstAst.qll b/ql/src/codeql_ql/printAstAst.qll
index 42603fe3752..feb525908d2 100644
--- a/ql/src/codeql_ql/printAstAst.qll
+++ b/ql/src/codeql_ql/printAstAst.qll
@@ -53,7 +53,7 @@ class PrintAstNode extends AstNode {
     result =
       any(int i |
         this =
-          rank[i](AstNode p, Location l, File f |
+          rank[i](PrintAstNode p, Location l, File f |
             l = p.getLocation() and
             f = l.getFile()
           |
diff --git a/ql/src/codeql_ql/printAstGenerated.qll b/ql/src/codeql_ql/printAstGenerated.qll
index f5bc1757209..e8029419713 100644
--- a/ql/src/codeql_ql/printAstGenerated.qll
+++ b/ql/src/codeql_ql/printAstGenerated.qll
@@ -31,7 +31,6 @@ class PrintAstConfiguration extends string {
  * The ordering is location based and pretty arbitary.
  */
 AstNode getAstChild(PrintAstNode parent, int i) {
-  parent.shouldPrint() and
   result =
     rank[i](AstNode child, Location l |
       child.getParent() = parent and
@@ -47,32 +46,24 @@ AstNode getAstChild(PrintAstNode parent, int i) {
  * A node in the output tree.
  */
 class PrintAstNode extends AstNode {
-  string getProperty(string key) {
-    this.shouldPrint() and
-    (
-      key = "semmle.label" and
-      result = "[" + concat(this.getAPrimaryQlClass(), ", ") + "] " + this.toString()
-      or
-      key = "semmle.order" and
-      result =
-        any(int i |
-          this =
-            rank[i](AstNode p, Location l, File f |
-              l = p.getLocation() and
-              f = l.getFile()
-            |
-              p order by f.getBaseName(), f.getAbsolutePath(), l.getStartLine(), l.getStartColumn()
-            )
-        ).toString()
-    )
-  }
+  PrintAstNode() { shouldPrintNode(this) }
 
-  /**
-   * Holds if this node should be printed in the output. By default, all nodes
-   * are printed, but the query can override
-   * `PrintAstConfiguration.shouldPrintNode` to filter the output.
-   */
-  predicate shouldPrint() { shouldPrintNode(this) }
+  string getProperty(string key) {
+    key = "semmle.label" and
+    result = "[" + concat(this.getAPrimaryQlClass(), ", ") + "] " + this.toString()
+    or
+    key = "semmle.order" and
+    result =
+      any(int i |
+        this =
+          rank[i](PrintAstNode p, Location l, File f |
+            l = p.getLocation() and
+            f = l.getFile()
+          |
+            p order by f.getBaseName(), f.getAbsolutePath(), l.getStartLine(), l.getStartColumn()
+          )
+      ).toString()
+  }
 
   /**
    * Gets the child node that is accessed using the predicate `edgeName`.
@@ -93,18 +84,13 @@ private predicate shouldPrintNode(AstNode n) {
  * Holds if `node` belongs to the output tree, and its property `key` has the
  * given `value`.
  */
-query predicate nodes(PrintAstNode node, string key, string value) {
-  node.shouldPrint() and
-  value = node.getProperty(key)
-}
+query predicate nodes(PrintAstNode node, string key, string value) { value = node.getProperty(key) }
 
 /**
  * Holds if `target` is a child of `source` in the AST, and property `key` of
  * the edge has the given `value`.
  */
 query predicate edges(PrintAstNode source, PrintAstNode target, string key, string value) {
-  source.shouldPrint() and
-  target.shouldPrint() and
   target = source.getChild(_) and
   (
     key = "semmle.label" and

From 12627d19fae9ba850726ef7be594fb1e5d08b8ca Mon Sep 17 00:00:00 2001
From: Erik Krogh Kristensen <erik-krogh@github.com>
Date: Mon, 31 May 2021 07:46:51 +0000
Subject: [PATCH 07/10] QL: improve scope resolution performance

---
 ql/src/codeql_ql/ast/internal/Variable.qll | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ql/src/codeql_ql/ast/internal/Variable.qll b/ql/src/codeql_ql/ast/internal/Variable.qll
index 89ff1288949..b5791f457ae 100644
--- a/ql/src/codeql_ql/ast/internal/Variable.qll
+++ b/ql/src/codeql_ql/ast/internal/Variable.qll
@@ -56,6 +56,7 @@ private AstNode parent(AstNode child) {
   not child instanceof VariableScope
 }
 
+pragma[nomagic]
 VariableScope scopeOf(AstNode n) { result = parent*(n.getParent()) }
 
 private string getName(Identifier i) {

From 3ee642868d80a53deb3a328afe674b7a154333c5 Mon Sep 17 00:00:00 2001
From: Erik Krogh Kristensen <erik-krogh@github.com>
Date: Mon, 31 May 2021 07:51:11 +0000
Subject: [PATCH 08/10] QL: improve performance of type resolution

---
 ql/src/codeql_ql/ast/internal/Type.qll | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/ql/src/codeql_ql/ast/internal/Type.qll b/ql/src/codeql_ql/ast/internal/Type.qll
index 3de6e420144..216adbe89ac 100644
--- a/ql/src/codeql_ql/ast/internal/Type.qll
+++ b/ql/src/codeql_ql/ast/internal/Type.qll
@@ -266,12 +266,14 @@ predicate resolveTypeExpr(TypeExpr te, Type t) {
     if primTypeName(te.getClassName())
     then t = TPrimitive(te.getClassName())
     else
-      exists(FileOrModule m, boolean public | qualifier(te, m, public) |
-        defines(m, te.getClassName(), t, public)
+      exists(FileOrModule m, boolean public, string clName | qualifier(te, m, public, clName) |
+        defines(m, clName, t, public)
       )
 }
 
-private predicate qualifier(TypeExpr te, FileOrModule m, boolean public) {
+pragma[noinline]
+private predicate qualifier(TypeExpr te, FileOrModule m, boolean public, string clName) {
+  te.getClassName() = clName and
   if exists(te.getModule())
   then (
     public = true and m = te.getModule().getResolvedModule()

From f2f6d34f210363cd9e1226d3822ea2fd6df7c78b Mon Sep 17 00:00:00 2001
From: Erik Krogh Kristensen <erik-krogh@github.com>
Date: Mon, 31 May 2021 08:06:49 +0000
Subject: [PATCH 09/10] QL: improve performance of module resolution

---
 ql/src/codeql_ql/ast/internal/Module.qll | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/ql/src/codeql_ql/ast/internal/Module.qll b/ql/src/codeql_ql/ast/internal/Module.qll
index 210e7f6b352..2a3eeaeca20 100644
--- a/ql/src/codeql_ql/ast/internal/Module.qll
+++ b/ql/src/codeql_ql/ast/internal/Module.qll
@@ -186,13 +186,21 @@ private module Cached {
   predicate resolveModuleExpr(ModuleExpr me, FileOrModule m) {
     not m = TFile(any(File f | f.getExtension() = "ql")) and
     not exists(me.getQualifier()) and
-    definesModule(getEnclosingModule(me).getEnclosing*(), me.getName(), m, _)
+    exists(ContainerOrModule enclosing, string name | resolveModuleExprHelper(me, enclosing, name) |
+      definesModule(enclosing.getEnclosing*(), name, m, _)
+    )
     or
     exists(FileOrModule mid |
       resolveModuleExpr(me.getQualifier(), mid) and
       definesModule(mid, me.getName(), m, true)
     )
   }
+
+  pragma[noinline]
+  private predicate resolveModuleExprHelper(ModuleExpr me, ContainerOrModule enclosing, string name) {
+    enclosing = getEnclosingModule(me) and
+    name = me.getName()
+  }
 }
 
 import Cached

From cbd0caa4ababb2d4ba85c9e5c832dca9d1492b97 Mon Sep 17 00:00:00 2001
From: Taus <tausbn@github.com>
Date: Mon, 31 May 2021 08:55:27 +0000
Subject: [PATCH 10/10] QL: Fix `getQLDoc` compilation error

---
 ql/src/codeql_ql/ast/Ast.qll | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ql/src/codeql_ql/ast/Ast.qll b/ql/src/codeql_ql/ast/Ast.qll
index 15c3c1f2f55..ab625234480 100644
--- a/ql/src/codeql_ql/ast/Ast.qll
+++ b/ql/src/codeql_ql/ast/Ast.qll
@@ -122,7 +122,7 @@ class TopLevel extends TTopLevel, AstNode {
 
   override string getAPrimaryQlClass() { result = "TopLevel" }
 
-  override QLDoc getQLDoc() { toGenerated(result) = file.getChild(0).getChild(0) }
+  override QLDoc getQLDoc() { result = this.getMember(0) }
 }
 
 class QLDoc extends TQLDoc, AstNode {