Merge pull request #302 from github/rm_tokeninfo_idx

Remove unused columns from tokeninfo tables

extractor/src/extractor.rs

@@ -190,7 +190,6 @@ pub fn extract(
         // TODO: should we handle path strings that are not valid UTF8 better?
         path: format!("{}", path.display()),
         file_label: *file_label,
-        token_counter: 0,
         toplevel_child_counter: 0,
         stack: Vec::new(),
         language_prefix,
@@ -300,8 +299,6 @@ struct Visitor<'a> {
     source: &'a Vec<u8>,
     /// A TrapWriter to accumulate trap entries
     trap_writer: &'a mut TrapWriter,
-    /// A counter for tokens
-    token_counter: usize,
     /// A counter for top-level child nodes
     toplevel_child_counter: usize,
     /// Language prefix
@@ -425,13 +422,10 @@ impl Visitor<'_> {
                     vec![
                         Arg::Label(id),
                         Arg::Int(*kind_id),
-                        Arg::Label(self.file_label),
-                        Arg::Int(self.token_counter),
                         sliced_source_arg(self.source, node),
                         Arg::Label(loc),
                     ],
                 );
-                self.token_counter += 1;
             }
             EntryKind::Table {
                 fields,

generator/src/main.rs

@@ -305,20 +305,6 @@ fn create_tokeninfo<'a>(name: &'a str, type_name: &'a str) -> dbscheme::Table<'a
                 ql_type: ql::Type::Int,
                 ql_type_is_ref: true,
             },
-            dbscheme::Column {
-                unique: false,
-                db_type: dbscheme::DbColumnType::Int,
-                name: "file",
-                ql_type: ql::Type::AtType("file"),
-                ql_type_is_ref: true,
-            },
-            dbscheme::Column {
-                unique: false,
-                db_type: dbscheme::DbColumnType::Int,
-                name: "idx",
-                ql_type: ql::Type::Int,
-                ql_type_is_ref: true,
-            },
             dbscheme::Column {
                 unique: false,
                 db_type: dbscheme::DbColumnType::String,

generator/src/ql_gen.rs

@@ -119,14 +119,14 @@ pub fn create_ast_node_class<'a>(ast_node: &'a str, ast_node_parent: &'a str) ->
 }
 
 pub fn create_token_class<'a>(token_type: &'a str, tokeninfo: &'a str) -> ql::Class<'a> {
-    let tokeninfo_arity = 6;
+    let tokeninfo_arity = 4;
     let get_value = ql::Predicate {
         qldoc: Some(String::from("Gets the value of this token.")),
         name: "getValue",
         overridden: false,
         return_type: Some(ql::Type::String),
         formal_parameters: vec![],
-        body: create_get_field_expr_for_column_storage("result", tokeninfo, 3, tokeninfo_arity),
+        body: create_get_field_expr_for_column_storage("result", tokeninfo, 1, tokeninfo_arity),
     };
     let get_location = ql::Predicate {
         qldoc: Some(String::from("Gets the location of this token.")),
@@ -134,7 +134,7 @@ pub fn create_token_class<'a>(token_type: &'a str, tokeninfo: &'a str) -> ql::Cl
         overridden: true,
         return_type: Some(ql::Type::Normal("Location")),
         formal_parameters: vec![],
-        body: create_get_field_expr_for_column_storage("result", tokeninfo, 4, tokeninfo_arity),
+        body: create_get_field_expr_for_column_storage("result", tokeninfo, 2, tokeninfo_arity),
     };
     let to_string = ql::Predicate {
         qldoc: Some(String::from(

ql/lib/codeql/ruby/ast/internal/TreeSitter.qll

@@ -34,10 +34,10 @@ module Ruby {
   /** A token. */
   class Token extends @ruby_token, AstNode {
     /** Gets the value of this token. */
-    string getValue() { ruby_tokeninfo(this, _, _, _, result, _) }
+    string getValue() { ruby_tokeninfo(this, _, result, _) }
 
     /** Gets the location of this token. */
-    override Location getLocation() { ruby_tokeninfo(this, _, _, _, _, result) }
+    override Location getLocation() { ruby_tokeninfo(this, _, _, result) }
 
     /** Gets a string representation of this element. */
     override string toString() { result = getValue() }
@@ -1875,10 +1875,10 @@ module Erb {
   /** A token. */
   class Token extends @erb_token, AstNode {
     /** Gets the value of this token. */
-    string getValue() { erb_tokeninfo(this, _, _, _, result, _) }
+    string getValue() { erb_tokeninfo(this, _, result, _) }
 
     /** Gets the location of this token. */
-    override Location getLocation() { erb_tokeninfo(this, _, _, _, _, result) }
+    override Location getLocation() { erb_tokeninfo(this, _, _, result) }
 
     /** Gets a string representation of this element. */
     override string toString() { result = getValue() }

ql/lib/ruby.dbscheme

@@ -1205,8 +1205,6 @@ ruby_yield_def(
 ruby_tokeninfo(
   unique int id: @ruby_token,
   int kind: int ref,
-  int file: @file ref,
-  int idx: int ref,
   string value: string ref,
   int loc: @location ref
 );
@@ -1293,8 +1291,6 @@ erb_template_def(
 erb_tokeninfo(
   unique int id: @erb_token,
   int kind: int ref,
-  int file: @file ref,
-  int idx: int ref,
   string value: string ref,
   int loc: @location ref
 );

ql/lib/upgrades/b5aef9c93ae64f848017d2dcb760eed916ab0cdd/upgrade.properties

@@ -0,0 +1,4 @@
+description: Removed unused columns from the `*_tokeninfo` relations
+compatibility: full
+ruby_tokeninfo.rel: reorder ruby_tokeninfo.rel (int id, int kind, int file, int idx, string value, int loc) id kind value loc
+erb_tokeninfo.rel: reorder erb_tokeninfo.rel (int id, int kind, int file, int idx, string value, int loc) id kind value loc