Merge pull request #20337 from d10c/d10c/python-overlay-compilation-plus-extractor

Python: enable overlay compilation + extractor overlay support

python/ql/consistency-queries/OverlayDiscardConsistency.ql

@@ -0,0 +1 @@
+import semmle.python.internal.OverlayDiscardConsistencyQuery

python/ql/lib/change-notes/2025-10-02-python-overlay-compilation-plus-extractor.md

@@ -0,0 +1,5 @@
+---
+category: feature
+---
+
+* Initial support for incremental Python databases via `codeql database create --overlay-base`/`--overlay-changes`.

python/ql/lib/python.qll

@@ -37,6 +37,7 @@ import semmle.python.pointsto.CallGraph
 import semmle.python.objects.ObjectAPI
 import semmle.python.Unit
 import site
+private import semmle.python.Overlay
 // Removing this import perturbs the compilation process enough that the points-to analysis gets
 // compiled -- and cached -- differently depending on whether the data flow library is imported. By
 // importing it privately here, we ensure that the points-to analysis is compiled the same way.

python/ql/lib/qlpack.yml

@@ -19,3 +19,4 @@ dataExtensions:
   - semmle/python/frameworks/**/*.model.yml
   - ext/*.model.yml
 warnOnImplicitThis: true
+compileForOverlayEval: true

python/ql/lib/semmle/python/Overlay.qll

@@ -0,0 +1,355 @@
+/**
+ * Defines entity discard predicates for Python overlay analysis.
+ */
+
+/*- Predicates -*/
+/**
+ * Holds always for the overlay variant and never for the base variant.
+ * This local predicate is used to define local predicates that behave
+ * differently for the base and overlay variant.
+ */
+overlay[local]
+predicate isOverlay() { databaseMetadata("isOverlay", "true") }
+
+overlay[local]
+private string getPathForLocation(@location loc) {
+  exists(@file file | locations_default(loc, file, _, _, _, _) | files(file, result))
+  or
+  exists(@py_Module mod | locations_ast(loc, mod, _, _, _, _) | result = getPathForModule(mod))
+}
+
+overlay[local]
+private string getPathForModule(@py_Module mod) {
+  exists(@container fileOrFolder | py_module_path(mod, fileOrFolder) |
+    result = getPathForContainer(fileOrFolder)
+  )
+}
+
+overlay[local]
+private string getPathForContainer(@container fileOrFolder) {
+  files(fileOrFolder, result) or folders(fileOrFolder, result)
+}
+
+/*- Discardable entities and their discard predicates -*/
+/** Python database entities that use named TRAP IDs; the rest use *-ids. */
+overlay[local]
+private class NamedEntity = @py_Module or @container or @py_cobject;
+
+overlay[discard_entity]
+private predicate discardNamedEntity(@top el) {
+  el instanceof NamedEntity and
+  // Entities with named IDs can exist both in base, overlay, or both.
+  exists(Discardable d | d = el |
+    overlayChangedFiles(d.getPath()) and
+    not d.existsInOverlay()
+  )
+}
+
+overlay[discard_entity]
+private predicate discardStarEntity(@top el) {
+  not el instanceof NamedEntity and
+  // Entities with *-ids can exist either in base or overlay, but not both.
+  exists(Discardable d | d = el |
+    overlayChangedFiles(d.getPath()) and
+    d.existsInBase()
+  )
+}
+
+/**
+ * An abstract base class for all elements that can be discarded from the base.
+ */
+overlay[local]
+abstract class Discardable extends @top {
+  /** Gets the path to the file in which this element occurs. */
+  abstract string getPath();
+
+  /** Holds if this element exists in the base variant. */
+  predicate existsInBase() { not isOverlay() and exists(this) }
+
+  /** Holds if this element exists in the overlay variant. */
+  predicate existsInOverlay() { isOverlay() and exists(this) }
+
+  /** Gets a textual representation of this discardable element. */
+  string toString() { none() }
+}
+
+/**
+ * Discardable locatable AST nodes (`@py_location_parent`).
+ */
+overlay[local]
+final private class DiscardableLocatable extends Discardable instanceof @py_location_parent {
+  override string getPath() {
+    exists(@location loc | py_locations(loc, this) | result = getPathForLocation(loc))
+  }
+}
+
+/**
+ * Discardable scopes (classes, functions, modules).
+ */
+overlay[local]
+final private class DiscardableScope extends Discardable instanceof @py_scope {
+  override string getPath() {
+    exists(@location loc | py_scope_location(loc, this) | result = getPathForLocation(loc))
+    or
+    result = getPathForModule(this)
+  }
+}
+
+/**
+ * Discardable files and folders.
+ */
+overlay[local]
+final private class DiscardableContainer extends Discardable instanceof @container {
+  override string getPath() { result = getPathForContainer(this) }
+}
+
+/** Discardable control flow nodes */
+overlay[local]
+final private class DiscardableCfgNode extends Discardable instanceof @py_flow_node {
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() | py_flow_bb_node(this, d.(@py_ast_node), _, _))
+  }
+}
+
+/** Discardable Python variables. */
+overlay[local]
+final private class DiscardableVar extends Discardable instanceof @py_variable {
+  override string getPath() {
+    exists(Discardable parent | result = parent.getPath() | variable(this, parent.(@py_scope), _))
+  }
+}
+
+/** Discardable SSA variables. */
+overlay[local]
+final private class DiscardableSsaVar extends Discardable instanceof @py_ssa_var {
+  override string getPath() {
+    exists(DiscardableVar other | result = other.getPath() | py_ssa_var(this, other))
+  }
+}
+
+/** Discardable locations. */
+overlay[local]
+final private class DiscardableLocation extends Discardable instanceof @location {
+  override string getPath() { result = getPathForLocation(this) }
+}
+
+/** Discardable lines. */
+overlay[local]
+final private class DiscardableLine extends Discardable instanceof @py_line {
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() | py_line_lengths(this, d.(@py_Module), _, _))
+  }
+}
+
+/** Discardable string part lists. */
+overlay[local]
+final private class DiscardableStringPartList extends Discardable instanceof @py_StringPart_list {
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() | py_StringPart_lists(this, d.(@py_Bytes_or_Str)))
+  }
+}
+
+/** Discardable alias */
+overlay[local]
+final private class DiscardableAlias extends Discardable instanceof @py_alias {
+  override string getPath() {
+    exists(DiscardableAliasList d | result = d.getPath() | py_aliases(this, d, _))
+  }
+}
+
+/** Discardable alias list */
+overlay[local]
+final private class DiscardableAliasList extends Discardable instanceof @py_alias_list {
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() | py_alias_lists(this, d.(@py_Import)))
+  }
+}
+
+/** Discardable arguments */
+overlay[local]
+final private class DiscardableArguments extends Discardable instanceof @py_arguments {
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() | py_arguments(this, d.(@py_arguments_parent)))
+  }
+}
+
+/** Discardable boolop */
+overlay[local]
+final private class DiscardableBoolOp extends Discardable instanceof @py_boolop {
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() | py_boolops(this, _, d.(@py_BoolExpr)))
+  }
+}
+
+/** Discardable cmpop */
+overlay[local]
+final private class DiscardableCmpOp extends Discardable instanceof @py_cmpop {
+  override string getPath() {
+    exists(DiscardableCmpOpList d | result = d.getPath() | py_cmpops(this, _, d, _))
+  }
+}
+
+/** Discardable cmpop list */
+overlay[local]
+final private class DiscardableCmpOpList extends Discardable instanceof @py_cmpop_list {
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() | py_cmpop_lists(this, d.(@py_Compare)))
+  }
+}
+
+/** Discardable comprehension list */
+overlay[local]
+final private class DiscardableComprehensionList extends Discardable instanceof @py_comprehension_list
+{
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() | py_comprehension_lists(this, d.(@py_ListComp)))
+  }
+}
+
+/** Discardable dict item list */
+overlay[local]
+final private class DiscardableDictItemList extends Discardable instanceof @py_dict_item_list {
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() |
+      py_dict_item_lists(this, d.(@py_dict_item_list_parent))
+    )
+  }
+}
+
+/** Discardable expr context */
+overlay[local]
+final private class DiscardableExprContext extends Discardable instanceof @py_expr_context {
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() |
+      py_expr_contexts(this, _, d.(@py_expr_context_parent))
+    )
+  }
+}
+
+/** Discardable expr list */
+overlay[local]
+final private class DiscardableExprList extends Discardable instanceof @py_expr_list {
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() | py_expr_lists(this, d.(@py_expr_list_parent), _))
+  }
+}
+
+/** Discardable operator */
+overlay[local]
+final private class DiscardableOperator extends Discardable instanceof @py_operator {
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() | py_operators(this, _, d.(@py_BinaryExpr)))
+  }
+}
+
+/** Discardable parameter list */
+overlay[local]
+final private class DiscardableParameterList extends Discardable instanceof @py_parameter_list {
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() | py_parameter_lists(this, d.(@py_Function)))
+  }
+}
+
+/** Discardable pattern list */
+overlay[local]
+final private class DiscardablePatternList extends Discardable instanceof @py_pattern_list {
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() |
+      py_pattern_lists(this, d.(@py_pattern_list_parent), _)
+    )
+  }
+}
+
+/** Discardable stmt list */
+overlay[local]
+final private class DiscardableStmtList extends Discardable instanceof @py_stmt_list {
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() | py_stmt_lists(this, d.(@py_stmt_list_parent), _))
+  }
+}
+
+/** Discardable str list */
+overlay[local]
+final private class DiscardableStrList extends Discardable instanceof @py_str_list {
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() | py_str_lists(this, d.(@py_str_list_parent)))
+  }
+}
+
+/** Discardable type parameter list */
+overlay[local]
+final private class DiscardableTypeParameterList extends Discardable instanceof @py_type_parameter_list
+{
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() |
+      py_type_parameter_lists(this, d.(@py_type_parameter_list_parent))
+    )
+  }
+}
+
+/** Discardable unaryop */
+overlay[local]
+final private class DiscardableUnaryOp extends Discardable instanceof @py_unaryop {
+  override string getPath() {
+    exists(Discardable d | result = d.getPath() | py_unaryops(this, _, d.(@py_UnaryExpr)))
+  }
+}
+
+/** Discardable comment */
+overlay[local]
+final private class DiscardableComment extends Discardable instanceof @py_comment {
+  override string getPath() {
+    exists(DiscardableLocation d | result = d.getPath() | py_comments(this, _, d))
+  }
+}
+
+/*- XML -*/
+overlay[local]
+final private class DiscardableXmlLocatable extends Discardable instanceof @xmllocatable {
+  override string getPath() {
+    exists(@location loc | xmllocations(this, loc) | result = getPathForLocation(loc))
+  }
+}
+
+overlay[local]
+private predicate overlayXmlExtracted(string path) {
+  exists(DiscardableXmlLocatable d | not files(d, _) and not xmlNs(d, _, _, _) |
+    d.existsInOverlay() and
+    path = d.getPath()
+  )
+}
+
+overlay[discard_entity]
+private predicate discardXmlLocatable(@xmllocatable el) {
+  exists(DiscardableXmlLocatable d | d = el |
+    // The XML extractor is currently not incremental and may extract more
+    // XML files than those included in `overlayChangedFiles`, so this discard predicate
+    // handles those files alongside the normal `discardStarEntity` logic.
+    overlayXmlExtracted(d.getPath()) and
+    d.existsInBase()
+  )
+}
+
+/*- YAML -*/
+overlay[local]
+final private class DiscardableYamlLocatable extends Discardable instanceof @yaml_locatable {
+  override string getPath() {
+    exists(@location loc | yaml_locations(this, loc) | result = getPathForLocation(loc))
+  }
+}
+
+overlay[local]
+private predicate overlayYamlExtracted(string path) {
+  exists(DiscardableYamlLocatable l | l.existsInOverlay() | path = l.getPath())
+}
+
+overlay[discard_entity]
+private predicate discardBaseYamlLocatable(@yaml_locatable el) {
+  exists(DiscardableYamlLocatable d | d = el |
+    // The Yaml extractor is currently not incremental and may extract more
+    // Yaml files than those included in `overlayChangedFiles`, so this discard predicate
+    // handles those files alongside the normal `discardStarEntity` logic.
+    overlayYamlExtracted(d.getPath()) and
+    d.existsInBase()
+  )
+}

python/ql/lib/semmle/python/internal/OverlayDiscardConsistencyQuery.qll

@@ -0,0 +1,90 @@
+/**
+ * Provides consistency queries for checking that every database entity
+ * that can be discarded (i.e. everything but `@py_cobject`) in an overlay
+ * database is indeed discarded, by proxy of having exactly one `Discardable.getPath()`.
+ */
+
+import python
+import semmle.python.Overlay
+
+class TopWithToString instanceof @top {
+  string getDbType() {
+    this instanceof @py_source_element and result = "@source_element"
+    or
+    this instanceof @py_object and result = "@py_object"
+    or
+    this instanceof @py_base_var and result = "@py_base_var"
+    or
+    this instanceof @location and result = "@location"
+    or
+    this instanceof @py_line and result = "@py_line"
+    or
+    this instanceof @py_comment and result = "@py_comment"
+    or
+    this instanceof @py_expr_parent and result = "@py_expr_parent"
+    or
+    this instanceof @py_expr_context and result = "@py_expr_context"
+    or
+    this instanceof @py_operator and result = "@py_operator"
+    or
+    this instanceof @py_boolop and result = "@py_boolop"
+    or
+    this instanceof @py_cmpop and result = "@py_cmpop"
+    or
+    this instanceof @py_unaryop and result = "@py_unaryop"
+    or
+    this instanceof @py_cmpop_list and result = "@py_cmpop_list"
+    or
+    this instanceof @py_alias_list and result = "@py_alias_list"
+    or
+    this instanceof @py_StringPart_list and result = "@py_StringPart_list"
+    or
+    this instanceof @py_comprehension_list and result = "@py_comprehension_list"
+    or
+    this instanceof @py_dict_item_list and result = "@py_dict_item_list"
+    or
+    this instanceof @py_pattern_list and result = "@py_pattern_list"
+    or
+    this instanceof @py_stmt_list and result = "@py_stmt_list"
+    or
+    this instanceof @py_str_list and result = "@py_str_list"
+    or
+    this instanceof @py_type_parameter_list and result = "@py_type_parameter_list"
+    or
+    this instanceof @externalDefect and result = "@externalDefect"
+    or
+    this instanceof @externalMetric and result = "@externalMetric"
+    or
+    this instanceof @externalDataElement and result = "@externalDataElement"
+    or
+    this instanceof @duplication_or_similarity and result = "@duplication_or_similarity"
+    or
+    this instanceof @svnentry and result = "@svnentry"
+    or
+    this instanceof @xmllocatable and result = "@xmllocatable"
+    or
+    this instanceof @yaml_locatable and result = "@yaml_locatable"
+  }
+
+  string toString() {
+    result = this.getDbType()
+    or
+    not exists(this.getDbType()) and
+    result = "Unknown type"
+  }
+}
+
+query predicate consistencyTest(TopWithToString el, string message) {
+  not el instanceof Discardable and
+  not el instanceof @py_cobject and // cannot be linked to a path
+  not el instanceof @externalDataElement and // cannot be linked to a path
+  message = "Not Discardable"
+  or
+  exists(Discardable d, int numPaths | d = el and numPaths = count(d.getPath()) |
+    numPaths = 0 and
+    message = "Discardable but no path found"
+    or
+    numPaths > 1 and
+    message = "Discardable but multiple paths found (" + concat(d.getPath(), ", ") + ")"
+  )
+}

python/ql/lib/semmlecode.python.dbscheme

@@ -47,6 +47,22 @@ externalData(
   string value : string ref
 );
 
+/*- Overlay support -*/
+
+/**
+ * The CLI will automatically emit the tuple `databaseMetadata("isOverlay", "true")`,
+ * along with an `overlayChangedFiles` tuple for each new/modified/deleted file,
+ * when building an overlay database, and these can be used by the discard predicates.
+ */
+databaseMetadata(
+  string metadataKey : string ref,
+  string value : string ref
+);
+
+overlayChangedFiles(
+  string path : string ref
+);
+
 /*- DEPRECATED: Snapshot date -*/
 
 snapshotDate(unique date snapshotDate : date ref);
@@ -1234,3 +1250,12 @@ py_decorated_object(int object : @py_object ref,
 @py_object = @py_cobject | @py_flow_node;
 
 @py_source_element = @py_ast_node | @container;
+
+/** The union of all Python database entities */
+@top =
+  @py_source_element | @py_object | @py_base_var | @location | @py_line | @py_comment |
+      @py_expr_parent | @py_expr_context |
+      @py_operator | @py_boolop | @py_cmpop | @py_unaryop |
+      @py_cmpop_list | @py_alias_list | @py_StringPart_list | @py_comprehension_list | @py_dict_item_list | @py_pattern_list | @py_stmt_list | @py_str_list | @py_type_parameter_list |
+      @externalDefect | @externalMetric | @externalDataElement | @duplication_or_similarity | @svnentry |
+      @xmllocatable | @yaml_locatable;

python/ql/lib/semmlecode.python.dbscheme.stats

@@ -1269,6 +1269,53 @@
 </dependencies>
 </relation>
 <relation>
+<name>databaseMetadata</name>
+<cardinality>1</cardinality>
+<columnsizes>
+<e>
+<k>metadataKey</k>
+<v>1</v>
+</e>
+<e>
+<k>value</k>
+<v>1</v>
+</e>
+</columnsizes>
+<dependencies>
+<dep>
+<src>metadataKey</src>
+<trg>value</trg>
+<val>
+<hist>
+<budget>12</budget>
+<bs/>
+</hist>
+</val>
+</dep>
+<dep>
+<src>value</src>
+<trg>metadataKey</trg>
+<val>
+<hist>
+<budget>12</budget>
+<bs/>
+</hist>
+</val>
+</dep>
+</dependencies>
+</relation>
+<relation>
+<name>overlayChangedFiles</name>
+<cardinality>50</cardinality>
+<columnsizes>
+<e>
+<k>path</k>
+<v>50</v>
+</e>
+</columnsizes>
+<dependencies/>
+</relation>
+<relation>
 <name>snapshotDate</name>
 <cardinality>2</cardinality>
 <columnsizes>

python/ql/lib/upgrades/5af903da088e3746aa283700a43a779302453523/upgrade.properties

@@ -0,0 +1,2 @@
+description: Add databaseMetadata and overlayChangedFiles relations
+compatibility: full

python/ql/lib/upgrades/6a1f497168da2f43828161d3c86db7d4c94c2b53/upgrade.properties

@@ -0,0 +1,2 @@
+description: Add @top type
+compatibility: full

python/ql/test/extractor-tests/overlay/basic-full-eval/greeting.py

@@ -0,0 +1,5 @@
+from new import *
+from lib import *
+
+def get_greeting():
+    return combine("Goodbye", get_new_target())

python/ql/test/extractor-tests/overlay/basic-full-eval/hello.py

@@ -0,0 +1,4 @@
+from greeting import *
+
+if __name__ == "__main__":
+    print(get_greeting())

python/ql/test/extractor-tests/overlay/basic-full-eval/lib/__init__.py

@@ -0,0 +1 @@
+def combine(a, b): return f"{a}, {b}!"

python/ql/test/extractor-tests/overlay/basic-full-eval/new.py

@@ -0,0 +1,2 @@
+def get_new_target():
+    return "世界"

python/ql/test/extractor-tests/overlay/basic-full-eval/options

@@ -0,0 +1 @@
+semmle-extractor-options: -R . -m hello.py --filter exclude:**/*.testproj/**

python/ql/test/extractor-tests/overlay/basic-full-eval/test.expected

@@ -0,0 +1,152 @@
+testStringLiterals
+| greeting.py:5:20:5:28 | StringLiteral | Goodbye |
+| hello.py:3:16:3:25 | StringLiteral | __main__ |
+| lib/__init__.py:1:27:1:29 | StringLiteral |  |
+| lib/__init__.py:1:31:1:34 | StringLiteral | ,  |
+| lib/__init__.py:1:36:1:38 | StringLiteral | ! |
+| new.py:2:12:2:15 | StringLiteral | \u4e16\u754c |
+testModules
+| greeting.py:0:0:0:0 | Module greeting |
+| hello.py:0:0:0:0 | Module hello |
+| lib/__init__.py:0:0:0:0 | Module lib.__init__ |
+| lib:0:0:0:0 | Package lib |
+| new.py:0:0:0:0 | Module new |
+testFunctions
+| greeting.py:4:1:4:19 | Function get_greeting |
+| lib/__init__.py:1:1:1:18 | Function combine |
+| new.py:1:1:1:21 | Function get_new_target |
+testClasses
+testLocations
+| greeting.py:0:0:0:0 | greeting.py:0 |
+| greeting.py:1:1:1:17 | greeting.py:1 |
+| greeting.py:1:6:1:8 | greeting.py:1 |
+| greeting.py:2:1:2:17 | greeting.py:2 |
+| greeting.py:2:6:2:8 | greeting.py:2 |
+| greeting.py:4:1:4:19 | greeting.py:4 |
+| greeting.py:4:1:4:19 | greeting.py:4 |
+| greeting.py:4:1:4:19 | greeting.py:4 |
+| greeting.py:4:5:4:16 | greeting.py:4 |
+| greeting.py:5:5:5:47 | greeting.py:5 |
+| greeting.py:5:12:5:18 | greeting.py:5 |
+| greeting.py:5:12:5:47 | greeting.py:5 |
+| greeting.py:5:20:5:28 | greeting.py:5 |
+| greeting.py:5:31:5:44 | greeting.py:5 |
+| greeting.py:5:31:5:46 | greeting.py:5 |
+| hello.py:0:0:0:0 | hello.py:0 |
+| hello.py:1:1:1:22 | hello.py:1 |
+| hello.py:1:6:1:13 | hello.py:1 |
+| hello.py:3:1:3:26 | hello.py:3 |
+| hello.py:3:4:3:11 | hello.py:3 |
+| hello.py:3:4:3:25 | hello.py:3 |
+| hello.py:3:16:3:25 | hello.py:3 |
+| hello.py:4:5:4:9 | hello.py:4 |
+| hello.py:4:5:4:25 | hello.py:4 |
+| hello.py:4:5:4:25 | hello.py:4 |
+| hello.py:4:11:4:22 | hello.py:4 |
+| hello.py:4:11:4:24 | hello.py:4 |
+| lib/__init__.py:0:0:0:0 | lib/__init__.py:0 |
+| lib/__init__.py:1:1:1:18 | lib/__init__.py:1 |
+| lib/__init__.py:1:1:1:18 | lib/__init__.py:1 |
+| lib/__init__.py:1:1:1:18 | lib/__init__.py:1 |
+| lib/__init__.py:1:5:1:11 | lib/__init__.py:1 |
+| lib/__init__.py:1:13:1:13 | lib/__init__.py:1 |
+| lib/__init__.py:1:16:1:16 | lib/__init__.py:1 |
+| lib/__init__.py:1:20:1:38 | lib/__init__.py:1 |
+| lib/__init__.py:1:27:1:29 | lib/__init__.py:1 |
+| lib/__init__.py:1:27:1:38 | lib/__init__.py:1 |
+| lib/__init__.py:1:30:1:30 | lib/__init__.py:1 |
+| lib/__init__.py:1:31:1:34 | lib/__init__.py:1 |
+| lib/__init__.py:1:35:1:35 | lib/__init__.py:1 |
+| lib/__init__.py:1:36:1:38 | lib/__init__.py:1 |
+| lib:0:0:0:0 | lib:0 |
+| new.py:0:0:0:0 | new.py:0 |
+| new.py:1:1:1:21 | new.py:1 |
+| new.py:1:1:1:21 | new.py:1 |
+| new.py:1:1:1:21 | new.py:1 |
+| new.py:1:5:1:18 | new.py:1 |
+| new.py:2:5:2:15 | new.py:2 |
+| new.py:2:12:2:15 | new.py:2 |
+testFiles
+| greeting.py:0:0:0:0 | greeting.py |
+| hello.py:0:0:0:0 | hello.py |
+| lib/__init__.py:0:0:0:0 | lib/__init__.py |
+| new.py:0:0:0:0 | new.py |
+testCfgNodes
+| greeting.py:0:0:0:0 | Entry node for Module greeting |
+| greeting.py:0:0:0:0 | Exit node for Module greeting |
+| greeting.py:1:1:1:17 | ControlFlowNode for from new import * |
+| greeting.py:1:6:1:8 | ControlFlowNode for ImportExpr |
+| greeting.py:2:1:2:17 | ControlFlowNode for from lib import * |
+| greeting.py:2:6:2:8 | ControlFlowNode for ImportExpr |
+| greeting.py:4:1:4:19 | ControlFlowNode for FunctionExpr |
+| greeting.py:4:1:4:19 | Entry node for Function get_greeting |
+| greeting.py:4:1:4:19 | Exit node for Function get_greeting |
+| greeting.py:4:5:4:16 | ControlFlowNode for get_greeting |
+| greeting.py:5:5:5:47 | ControlFlowNode for Return |
+| greeting.py:5:12:5:18 | ControlFlowNode for combine |
+| greeting.py:5:12:5:47 | ControlFlowNode for combine() |
+| greeting.py:5:20:5:28 | ControlFlowNode for StringLiteral |
+| greeting.py:5:31:5:44 | ControlFlowNode for get_new_target |
+| greeting.py:5:31:5:46 | ControlFlowNode for get_new_target() |
+| hello.py:0:0:0:0 | Entry node for Module hello |
+| hello.py:0:0:0:0 | Exit node for Module hello |
+| hello.py:1:1:1:22 | ControlFlowNode for from greeting import * |
+| hello.py:1:6:1:13 | ControlFlowNode for ImportExpr |
+| hello.py:3:4:3:11 | ControlFlowNode for __name__ |
+| hello.py:3:4:3:25 | ControlFlowNode for Compare |
+| hello.py:3:16:3:25 | ControlFlowNode for StringLiteral |
+| hello.py:4:5:4:9 | ControlFlowNode for print |
+| hello.py:4:5:4:25 | ControlFlowNode for print() |
+| hello.py:4:11:4:22 | ControlFlowNode for get_greeting |
+| hello.py:4:11:4:24 | ControlFlowNode for get_greeting() |
+| lib/__init__.py:0:0:0:0 | Entry node for Module lib.__init__ |
+| lib/__init__.py:0:0:0:0 | Exit node for Module lib.__init__ |
+| lib/__init__.py:1:1:1:18 | ControlFlowNode for FunctionExpr |
+| lib/__init__.py:1:1:1:18 | Entry node for Function combine |
+| lib/__init__.py:1:1:1:18 | Exit node for Function combine |
+| lib/__init__.py:1:5:1:11 | ControlFlowNode for combine |
+| lib/__init__.py:1:13:1:13 | ControlFlowNode for a |
+| lib/__init__.py:1:16:1:16 | ControlFlowNode for b |
+| lib/__init__.py:1:20:1:38 | ControlFlowNode for Return |
+| lib/__init__.py:1:27:1:29 | ControlFlowNode for StringLiteral |
+| lib/__init__.py:1:27:1:38 | ControlFlowNode for Fstring |
+| lib/__init__.py:1:30:1:30 | ControlFlowNode for a |
+| lib/__init__.py:1:31:1:34 | ControlFlowNode for StringLiteral |
+| lib/__init__.py:1:35:1:35 | ControlFlowNode for b |
+| lib/__init__.py:1:36:1:38 | ControlFlowNode for StringLiteral |
+| lib:0:0:0:0 | Entry node for Package lib |
+| new.py:0:0:0:0 | Entry node for Module new |
+| new.py:0:0:0:0 | Exit node for Module new |
+| new.py:1:1:1:21 | ControlFlowNode for FunctionExpr |
+| new.py:1:1:1:21 | Entry node for Function get_new_target |
+| new.py:1:1:1:21 | Exit node for Function get_new_target |
+| new.py:1:5:1:18 | ControlFlowNode for get_new_target |
+| new.py:2:5:2:15 | ControlFlowNode for Return |
+| new.py:2:12:2:15 | ControlFlowNode for StringLiteral |
+testSsaVars
+| file://:0:0:0:0 | SSA Variable __name__ |
+| file://:0:0:0:0 | SSA Variable get_greeting |
+| file://:0:0:0:0 | SSA Variable print |
+| greeting.py:4:5:4:16 | SSA Variable get_greeting |
+| lib/__init__.py:1:5:1:11 | SSA Variable combine |
+| lib/__init__.py:1:13:1:13 | SSA Variable a |
+| lib/__init__.py:1:16:1:16 | SSA Variable b |
+| new.py:1:5:1:18 | SSA Variable get_new_target |
+testVars
+| Global Variable __name__ | greeting.py:0:0:0:0 | Module greeting |
+| Global Variable __name__ | hello.py:0:0:0:0 | Module hello |
+| Global Variable __name__ | lib/__init__.py:0:0:0:0 | Module lib.__init__ |
+| Global Variable __name__ | new.py:0:0:0:0 | Module new |
+| Global Variable __package__ | greeting.py:0:0:0:0 | Module greeting |
+| Global Variable __package__ | hello.py:0:0:0:0 | Module hello |
+| Global Variable __package__ | lib/__init__.py:0:0:0:0 | Module lib.__init__ |
+| Global Variable __package__ | new.py:0:0:0:0 | Module new |
+| Global Variable combine | greeting.py:0:0:0:0 | Module greeting |
+| Global Variable combine | lib/__init__.py:0:0:0:0 | Module lib.__init__ |
+| Global Variable get_greeting | greeting.py:0:0:0:0 | Module greeting |
+| Global Variable get_greeting | hello.py:0:0:0:0 | Module hello |
+| Global Variable get_new_target | greeting.py:0:0:0:0 | Module greeting |
+| Global Variable get_new_target | new.py:0:0:0:0 | Module new |
+| Global Variable print | hello.py:0:0:0:0 | Module hello |
+| Local Variable a | lib/__init__.py:1:1:1:18 | Function combine |
+| Local Variable b | lib/__init__.py:1:1:1:18 | Function combine |

python/ql/test/extractor-tests/overlay/basic-full-eval/test.ql

@@ -0,0 +1,19 @@
+import python
+
+query predicate testStringLiterals(StringLiteral l, string text) { l.getText() = text }
+
+query predicate testModules(Module m) { any() }
+
+query predicate testFunctions(Function f) { any() }
+
+query predicate testClasses(Class c) { any() }
+
+query predicate testLocations(Location l) { any() }
+
+query predicate testFiles(File f) { any() }
+
+query predicate testCfgNodes(ControlFlowNode n) { any() }
+
+query predicate testSsaVars(SsaVariable var) { any() }
+
+query predicate testVars(Variable var, Scope s) { s = var.getScope() }

python/ql/test/extractor-tests/overlay/basic-overlay-eval/codeql-test.yml

@@ -0,0 +1,3 @@
+overlay:
+  base: "orig_src"
+  overlay: "../basic-full-eval"

python/ql/test/extractor-tests/overlay/basic-overlay-eval/options

@@ -0,0 +1 @@
+semmle-extractor-options: -R . -m hello.py --filter exclude:**/*.testproj/**

python/ql/test/extractor-tests/overlay/basic-overlay-eval/orig_src/greeting.py

@@ -0,0 +1,5 @@
+from old import *
+from lib import *
+
+def get_greeting():
+    return combine("Hello", get_old_target())

python/ql/test/extractor-tests/overlay/basic-overlay-eval/orig_src/hello.py

@@ -0,0 +1,4 @@
+from greeting import *
+
+if __name__ == "__main__":
+    print(get_greeting())

python/ql/test/extractor-tests/overlay/basic-overlay-eval/orig_src/lib/__init__.py

@@ -0,0 +1 @@
+def combine(a, b): return f"{a}, {b}!"

python/ql/test/extractor-tests/overlay/basic-overlay-eval/orig_src/old.py

@@ -0,0 +1,2 @@
+def get_old_target():
+    return "World"

python/ql/test/extractor-tests/overlay/basic-overlay-eval/test.qlref

@@ -0,0 +1 @@
+../basic-full-eval/test.ql