Merge pull request #19896 from github/nickrolfe/overlay-deleted-files

Java/Ruby/Rust/QL: add `overlayChangedFiles` relation to dbscheme

java/downgrades/1b8f5f4c747e4249f4731796ccaa0661c7434d8a/upgrade.properties

@@ -0,0 +1,3 @@
+description: Add overlayChangedFiles relation
+compatibility: full
+overlayChangedFiles.rel: delete

java/ql/lib/config/semmlecode.dbscheme

@@ -211,6 +211,10 @@ databaseMetadata(
   string value : string ref
 );
 
+overlayChangedFiles(
+  string path: string ref
+);
+
 /*
  * SMAP
  */

java/ql/lib/config/semmlecode.dbscheme.stats

@@ -4004,6 +4004,17 @@
                 </dep>
             </dependencies>
         </relation>
+        <relation>
+            <name>overlayChangedFiles</name>
+            <cardinality>50</cardinality>
+            <columnsizes>
+                <e>
+                    <k>path</k>
+                    <v>50</v>
+                </e>
+            </columnsizes>
+            <dependencies/>
+        </relation>
         <relation>
             <name>smap_header</name>
             <cardinality>1</cardinality>

java/ql/lib/upgrades/38d02c063878000356a3e5db49d5a6a8f38efe24/upgrade.properties

@@ -0,0 +1,2 @@
+description: Add overlayChangedFiles relation
+compatibility: full

ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll

@@ -60,10 +60,6 @@ module QL {
     )
   }
 
-  /** Holds if `file` was extracted as part of the overlay database. */
-  overlay[local]
-  private predicate discardFile(@file file) { isOverlay() and file = getNodeFile(_) }
-
   /** Holds if `node` is in the `file` and is part of the overlay base database. */
   overlay[local]
   private predicate discardableAstNode(@file file, @ql_ast_node node) {
@@ -73,7 +69,9 @@ module QL {
   /** Holds if `node` should be discarded, because it is part of the overlay base and is in a file that was also extracted as part of the overlay database. */
   overlay[discard_entity]
   private predicate discardAstNode(@ql_ast_node node) {
-    exists(@file file | discardableAstNode(file, node) and discardFile(file))
+    exists(@file file, string path | files(file, path) |
+      discardableAstNode(file, node) and overlayChangedFiles(path)
+    )
   }
 
   /** A class representing `add_expr` nodes. */
@@ -1354,10 +1352,6 @@ module Dbscheme {
     )
   }
 
-  /** Holds if `file` was extracted as part of the overlay database. */
-  overlay[local]
-  private predicate discardFile(@file file) { isOverlay() and file = getNodeFile(_) }
-
   /** Holds if `node` is in the `file` and is part of the overlay base database. */
   overlay[local]
   private predicate discardableAstNode(@file file, @dbscheme_ast_node node) {
@@ -1367,7 +1361,9 @@ module Dbscheme {
   /** Holds if `node` should be discarded, because it is part of the overlay base and is in a file that was also extracted as part of the overlay database. */
   overlay[discard_entity]
   private predicate discardAstNode(@dbscheme_ast_node node) {
-    exists(@file file | discardableAstNode(file, node) and discardFile(file))
+    exists(@file file, string path | files(file, path) |
+      discardableAstNode(file, node) and overlayChangedFiles(path)
+    )
   }
 
   /** A class representing `annotName` tokens. */
@@ -1714,10 +1710,6 @@ module Blame {
     )
   }
 
-  /** Holds if `file` was extracted as part of the overlay database. */
-  overlay[local]
-  private predicate discardFile(@file file) { isOverlay() and file = getNodeFile(_) }
-
   /** Holds if `node` is in the `file` and is part of the overlay base database. */
   overlay[local]
   private predicate discardableAstNode(@file file, @blame_ast_node node) {
@@ -1727,7 +1719,9 @@ module Blame {
   /** Holds if `node` should be discarded, because it is part of the overlay base and is in a file that was also extracted as part of the overlay database. */
   overlay[discard_entity]
   private predicate discardAstNode(@blame_ast_node node) {
-    exists(@file file | discardableAstNode(file, node) and discardFile(file))
+    exists(@file file, string path | files(file, path) |
+      discardableAstNode(file, node) and overlayChangedFiles(path)
+    )
   }
 
   /** A class representing `blame_entry` nodes. */
@@ -1851,10 +1845,6 @@ module JSON {
     )
   }
 
-  /** Holds if `file` was extracted as part of the overlay database. */
-  overlay[local]
-  private predicate discardFile(@file file) { isOverlay() and file = getNodeFile(_) }
-
   /** Holds if `node` is in the `file` and is part of the overlay base database. */
   overlay[local]
   private predicate discardableAstNode(@file file, @json_ast_node node) {
@@ -1864,7 +1854,9 @@ module JSON {
   /** Holds if `node` should be discarded, because it is part of the overlay base and is in a file that was also extracted as part of the overlay database. */
   overlay[discard_entity]
   private predicate discardAstNode(@json_ast_node node) {
-    exists(@file file | discardableAstNode(file, node) and discardFile(file))
+    exists(@file file, string path | files(file, path) |
+      discardableAstNode(file, node) and overlayChangedFiles(path)
+    )
   }
 
   class UnderscoreValue extends @json_underscore_value, AstNode { }

ql/ql/src/ql.dbscheme

@@ -114,6 +114,10 @@ databaseMetadata(
   string value: string ref
 );
 
+overlayChangedFiles(
+  string path: string ref
+);
+
 /*- QL dbscheme -*/
 @ql_add_expr_left_type = @ql_add_expr | @ql_aggregate | @ql_call_or_unqual_agg_expr | @ql_comp_term | @ql_conjunction | @ql_disjunction | @ql_expr_annotation | @ql_if_term | @ql_implication | @ql_in_expr | @ql_instance_of | @ql_literal | @ql_mul_expr | @ql_negation | @ql_par_expr | @ql_prefix_cast | @ql_qualified_expr | @ql_quantified | @ql_range | @ql_set_literal | @ql_special_call | @ql_super_ref | @ql_unary_expr | @ql_variable
 

ql/ql/src/ql.dbscheme.stats

@@ -22472,5 +22472,16 @@
                 </dep>
             </dependencies>
         </relation>
+        <relation>
+            <name>overlayChangedFiles</name>
+            <cardinality>50</cardinality>
+            <columnsizes>
+                <e>
+                    <k>path</k>
+                    <v>50</v>
+                </e>
+            </columnsizes>
+            <dependencies/>
+        </relation>
         </stats>
 </dbstats>

ruby/codeql-extractor.yml

@@ -3,7 +3,7 @@ display_name: "Ruby"
 version: 0.1.0
 column_kind: "utf8"
 legacy_qltest_extraction: true
-overlay_support_version: 20250108
+overlay_support_version: 20250626
 build_modes:
   - none
 github_api_languages:

ruby/downgrades/eae6926f5000373d5eb1d82131e1ff6152b67b2c/upgrade.properties

@@ -0,0 +1,3 @@
+description: Add overlayChangedFiles relation
+compatibility: full
+overlayChangedFiles.rel: delete

ruby/ql/lib/codeql/ruby/ast/internal/TreeSitter.qll

@@ -60,10 +60,6 @@ module Ruby {
     )
   }
 
-  /** Holds if `file` was extracted as part of the overlay database. */
-  overlay[local]
-  private predicate discardFile(@file file) { isOverlay() and file = getNodeFile(_) }
-
   /** Holds if `node` is in the `file` and is part of the overlay base database. */
   overlay[local]
   private predicate discardableAstNode(@file file, @ruby_ast_node node) {
@@ -73,7 +69,9 @@ module Ruby {
   /** Holds if `node` should be discarded, because it is part of the overlay base and is in a file that was also extracted as part of the overlay database. */
   overlay[discard_entity]
   private predicate discardAstNode(@ruby_ast_node node) {
-    exists(@file file | discardableAstNode(file, node) and discardFile(file))
+    exists(@file file, string path | files(file, path) |
+      discardableAstNode(file, node) and overlayChangedFiles(path)
+    )
   }
 
   class UnderscoreArg extends @ruby_underscore_arg, AstNode { }
@@ -2006,10 +2004,6 @@ module Erb {
     )
   }
 
-  /** Holds if `file` was extracted as part of the overlay database. */
-  overlay[local]
-  private predicate discardFile(@file file) { isOverlay() and file = getNodeFile(_) }
-
   /** Holds if `node` is in the `file` and is part of the overlay base database. */
   overlay[local]
   private predicate discardableAstNode(@file file, @erb_ast_node node) {
@@ -2019,7 +2013,9 @@ module Erb {
   /** Holds if `node` should be discarded, because it is part of the overlay base and is in a file that was also extracted as part of the overlay database. */
   overlay[discard_entity]
   private predicate discardAstNode(@erb_ast_node node) {
-    exists(@file file | discardableAstNode(file, node) and discardFile(file))
+    exists(@file file, string path | files(file, path) |
+      discardableAstNode(file, node) and overlayChangedFiles(path)
+    )
   }
 
   /** A class representing `code` tokens. */

ruby/ql/lib/ruby.dbscheme

@@ -114,6 +114,10 @@ databaseMetadata(
   string value: string ref
 );
 
+overlayChangedFiles(
+  string path: string ref
+);
+
 /*- Ruby dbscheme -*/
 @ruby_underscore_arg = @ruby_assignment | @ruby_binary | @ruby_conditional | @ruby_operator_assignment | @ruby_range | @ruby_unary | @ruby_underscore_primary
 

ruby/ql/lib/ruby.dbscheme.stats

@@ -21557,6 +21557,17 @@
                 </dep>
             </dependencies>
         </relation>
+        <relation>
+            <name>overlayChangedFiles</name>
+            <cardinality>50</cardinality>
+            <columnsizes>
+                <e>
+                    <k>path</k>
+                    <v>50</v>
+                </e>
+            </columnsizes>
+            <dependencies/>
+        </relation>
         <relation>
             <name>yaml_aliases</name>
             <cardinality>0</cardinality>

ruby/ql/lib/upgrades/dc51d416301df12df5b70fbc4338de6cc1f82bfd/upgrade.properties

@@ -0,0 +1,2 @@
+description: Add overlayChangedFiles relation
+compatibility: full

rust/downgrades/319c933d9615ccf40f363548cafd51d08c74a534/upgrade.properties

@@ -0,0 +1,3 @@
+description: Add overlayChangedFiles relation
+compatibility: full
+overlayChangedFiles.rel: delete

rust/ql/lib/rust.dbscheme

@@ -114,6 +114,10 @@ databaseMetadata(
   string value: string ref
 );
 
+overlayChangedFiles(
+  string path: string ref
+);
+
 
 // from prefix.dbscheme
 #keyset[id]

rust/ql/lib/upgrades/8e9b0c516ae822286689672d1020707020128a19/upgrade.properties

@@ -0,0 +1,2 @@
+description: Add overlayChangedFiles relation
+compatibility: full

shared/tree-sitter-extractor/src/generator/mod.rs

@@ -104,9 +104,6 @@ pub fn generate(
         body.push(ql::TopLevel::Predicate(
             ql_gen::create_get_node_file_predicate(&ast_node_name, &node_location_table_name),
         ));
-        body.push(ql::TopLevel::Predicate(
-            ql_gen::create_discard_file_predicate(),
-        ));
         body.push(ql::TopLevel::Predicate(
             ql_gen::create_discardable_ast_node_predicate(&ast_node_name),
         ));

shared/tree-sitter-extractor/src/generator/prefix.dbscheme

@@ -110,3 +110,7 @@ databaseMetadata(
   string metadataKey: string ref,
   string value: string ref
 );
+
+overlayChangedFiles(
+  string path: string ref
+);

shared/tree-sitter-extractor/src/generator/ql_gen.rs

@@ -318,34 +318,6 @@ pub fn create_get_node_file_predicate<'a>(
     }
 }
 
-pub fn create_discard_file_predicate<'a>() -> ql::Predicate<'a> {
-    ql::Predicate {
-        name: "discardFile",
-        qldoc: Some(String::from(
-            "Holds if `file` was extracted as part of the overlay database.",
-        )),
-        overridden: false,
-        is_private: true,
-        is_final: false,
-        overlay: Some(ql::OverlayAnnotation::Local),
-        return_type: None,
-        formal_parameters: vec![ql::FormalParameter {
-            name: "file",
-            param_type: ql::Type::At("file"),
-        }],
-        body: ql::Expression::And(vec![
-            ql::Expression::Pred("isOverlay", vec![]),
-            ql::Expression::Equals(
-                Box::new(ql::Expression::Var("file")),
-                Box::new(ql::Expression::Pred(
-                    "getNodeFile",
-                    vec![ql::Expression::Var("_")],
-                )),
-            ),
-        ]),
-    }
-}
-
 pub fn create_discardable_ast_node_predicate(ast_node_name: &str) -> ql::Predicate {
     ql::Predicate {
         name: "discardableAstNode",
@@ -398,17 +370,26 @@ pub fn create_discard_ast_node_predicate(ast_node_name: &str) -> ql::Predicate {
         }],
         body: ql::Expression::Aggregate {
             name: "exists",
-            vars: vec![ql::FormalParameter {
-                name: "file",
-                param_type: ql::Type::At("file"),
-            }],
-            range: None,
+            vars: vec![
+                ql::FormalParameter {
+                    name: "file",
+                    param_type: ql::Type::At("file"),
+                },
+                ql::FormalParameter {
+                    name: "path",
+                    param_type: ql::Type::String,
+                },
+            ],
+            range: Some(Box::new(ql::Expression::Pred(
+                "files",
+                vec![ql::Expression::Var("file"), ql::Expression::Var("path")],
+            ))),
             expr: Box::new(ql::Expression::And(vec![
                 ql::Expression::Pred(
                     "discardableAstNode",
                     vec![ql::Expression::Var("file"), ql::Expression::Var("node")],
                 ),
-                ql::Expression::Pred("discardFile", vec![ql::Expression::Var("file")]),
+                ql::Expression::Pred("overlayChangedFiles", vec![ql::Expression::Var("path")]),
             ])),
             second_expr: None,
         },