Merge branch 'main' into python/captured-variables-for-typetracking

python/ql/lib/change-notes/2023-04-20-yaml.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Added support for querying the contents of YAML files.

python/ql/lib/qlpack.yml

@@ -9,5 +9,6 @@ dependencies:
   codeql/regex: ${workspace}
   codeql/tutorial: ${workspace}
   codeql/util: ${workspace}
+  codeql/yaml: ${workspace}
 dataExtensions:
   - semmle/python/frameworks/**/model.yml

python/ql/lib/semmle/python/PrintAst.qll

@@ -8,6 +8,7 @@
 
 import python
 import semmle.python.RegexTreeView
+import semmle.python.Yaml
 
 private newtype TPrintAstConfiguration = MkPrintAstConfiguration()
 
@@ -53,7 +54,9 @@ private newtype TPrintAstNode =
     shouldPrint(list.getAnItem(), _) and
     not list = any(Module mod).getBody() and
     not forall(AstNode child | child = list.getAnItem() | isNotNeeded(child))
-  }
+  } or
+  TYamlNode(YamlNode node) or
+  TYamlMappingNode(YamlMapping mapping, int i) { exists(mapping.getKeyNode(i)) }
 
 /**
  * A node in the output tree.
@@ -633,6 +636,80 @@ private module PrettyPrinting {
   }
 }
 
+/**
+ * Classes for printing YAML AST.
+ */
+module PrintYaml {
+  /**
+   * A print node representing a YAML value in a .yml file.
+   */
+  class YamlNodeNode extends PrintAstNode, TYamlNode {
+    YamlNode node;
+
+    YamlNodeNode() { this = TYamlNode(node) }
+
+    override string toString() {
+      result = "[" + concat(node.getAPrimaryQlClass(), ",") + "] " + node.toString()
+    }
+
+    override Location getLocation() { result = node.getLocation() }
+
+    /**
+     * Gets the `YAMLNode` represented by this node.
+     */
+    final YamlNode getValue() { result = node }
+
+    override PrintAstNode getChild(int childIndex) {
+      exists(YamlNode child | result.(YamlNodeNode).getValue() = child |
+        child = node.getChildNode(childIndex)
+      )
+    }
+  }
+
+  /**
+   * A print node representing a `YAMLMapping`.
+   *
+   * Each child of this node aggregates the key and value of a mapping.
+   */
+  class YamlMappingNode extends YamlNodeNode {
+    override YamlMapping node;
+
+    override PrintAstNode getChild(int childIndex) {
+      exists(YamlMappingMapNode map | map = result | map.maps(node, childIndex))
+    }
+  }
+
+  /**
+   * A print node representing the `i`th mapping in `mapping`.
+   */
+  class YamlMappingMapNode extends PrintAstNode, TYamlMappingNode {
+    YamlMapping mapping;
+    int i;
+
+    YamlMappingMapNode() { this = TYamlMappingNode(mapping, i) }
+
+    override string toString() {
+      result = "(Mapping " + i + ")" and not exists(mapping.getKeyNode(i).(YamlScalar).getValue())
+      or
+      result = "(Mapping " + i + ") " + mapping.getKeyNode(i).(YamlScalar).getValue() + ":"
+    }
+
+    /**
+     * Holds if this print node represents the `index`th mapping of `m`.
+     */
+    predicate maps(YamlMapping m, int index) {
+      m = mapping and
+      index = i
+    }
+
+    override PrintAstNode getChild(int childIndex) {
+      childIndex = 0 and result.(YamlNodeNode).getValue() = mapping.getKeyNode(i)
+      or
+      childIndex = 1 and result.(YamlNodeNode).getValue() = mapping.getValueNode(i)
+    }
+  }
+}
+
 /** Holds if `node` belongs to the output tree, and its property `key` has the given `value`. */
 query predicate nodes(PrintAstNode node, string key, string value) { value = node.getProperty(key) }
 

python/ql/lib/semmle/python/Yaml.qll

@@ -0,0 +1,50 @@
+/**
+ * Provides classes for working with YAML data.
+ *
+ * YAML documents are represented as abstract syntax trees whose nodes
+ * are either YAML values or alias nodes referring to another YAML value.
+ */
+
+private import codeql.yaml.Yaml as LibYaml
+
+private module YamlSig implements LibYaml::InputSig {
+  import semmle.python.Files
+
+  class LocatableBase extends @yaml_locatable {
+    Location getLocation() { yaml_locations(this, result) }
+
+    string toString() { none() }
+  }
+
+  class NodeBase extends LocatableBase, @yaml_node {
+    NodeBase getChildNode(int i) { yaml(result, _, this, i, _, _) }
+
+    string getTag() { yaml(this, _, _, _, result, _) }
+
+    string getAnchor() { yaml_anchors(this, result) }
+
+    override string toString() { yaml(this, _, _, _, _, result) }
+  }
+
+  class ScalarNodeBase extends NodeBase, @yaml_scalar_node {
+    int getStyle() { yaml_scalars(this, result, _) }
+
+    string getValue() { yaml_scalars(this, _, result) }
+  }
+
+  class CollectionNodeBase extends NodeBase, @yaml_collection_node { }
+
+  class MappingNodeBase extends CollectionNodeBase, @yaml_mapping_node { }
+
+  class SequenceNodeBase extends CollectionNodeBase, @yaml_sequence_node { }
+
+  class AliasNodeBase extends NodeBase, @yaml_alias_node {
+    string getTarget() { yaml_aliases(this, result) }
+  }
+
+  class ParseErrorBase extends LocatableBase, @yaml_error {
+    string getMessage() { yaml_errors(this, result) }
+  }
+}
+
+import LibYaml::Make<YamlSig>

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl.qll

@@ -3031,6 +3031,17 @@ module Impl<FullStateConfigSig Config> {
       this instanceof PathNodeSinkGroup
     }
 
+    private string ppType() {
+      this instanceof PathNodeSink and result = ""
+      or
+      this.(PathNodeMid).getAp() instanceof AccessPathNil and result = ""
+      or
+      exists(DataFlowType t | t = this.(PathNodeMid).getAp().getHead().getContainerType() |
+        // The `concat` becomes "" if `ppReprType` has no result.
+        result = concat(" : " + ppReprType(t))
+      )
+    }
+
     private string ppAp() {
       this instanceof PathNodeSink and result = ""
       or
@@ -3046,14 +3057,14 @@ module Impl<FullStateConfigSig Config> {
     }
 
     /** Gets a textual representation of this element. */
-    string toString() { result = this.getNodeEx().toString() + this.ppAp() }
+    string toString() { result = this.getNodeEx().toString() + this.ppType() + this.ppAp() }
 
     /**
      * Gets a textual representation of this element, including a textual
      * representation of the call context.
      */
     string toStringWithContext() {
-      result = this.getNodeEx().toString() + this.ppAp() + this.ppCtx()
+      result = this.getNodeEx().toString() + this.ppType() + this.ppAp() + this.ppCtx()
     }
 
     /**
@@ -3998,14 +4009,14 @@ module Impl<FullStateConfigSig Config> {
      */
     class PartialPathNode extends TPartialPathNode {
       /** Gets a textual representation of this element. */
-      string toString() { result = this.getNodeEx().toString() + this.ppAp() }
+      string toString() { result = this.getNodeEx().toString() + this.ppType() + this.ppAp() }
 
       /**
        * Gets a textual representation of this element, including a textual
        * representation of the call context.
        */
       string toStringWithContext() {
-        result = this.getNodeEx().toString() + this.ppAp() + this.ppCtx()
+        result = this.getNodeEx().toString() + this.ppType() + this.ppAp() + this.ppCtx()
       }
 
       /**
@@ -4046,6 +4057,19 @@ module Impl<FullStateConfigSig Config> {
        */
       int getSinkDistance() { result = distSink(this.getNodeEx().getEnclosingCallable()) }
 
+      private string ppType() {
+        this instanceof PartialPathNodeRev and result = ""
+        or
+        this.(PartialPathNodeFwd).getAp() instanceof PartialAccessPathNil and result = ""
+        or
+        exists(DataFlowType t |
+          t = this.(PartialPathNodeFwd).getAp().(PartialAccessPathCons).getType()
+        |
+          // The `concat` becomes "" if `ppReprType` has no result.
+          result = concat(" : " + ppReprType(t))
+        )
+      }
+
       private string ppAp() {
         exists(string s |
           s = this.(PartialPathNodeFwd).getAp().toString() or

python/ql/lib/semmlecode.python.dbscheme

@@ -1104,3 +1104,44 @@ xmllocations(int xmlElement: @xmllocatable ref,
              int location: @location_default ref);
 
 @xmllocatable = @xmlcharacters | @xmlelement | @xmlcomment | @xmlattribute | @xmldtd | @file | @xmlnamespace;
+
+/**
+ *  YAML
+ */
+
+#keyset[parent, idx]
+yaml (unique int id: @yaml_node,
+      int kind: int ref,
+      int parent: @yaml_node_parent ref,
+      int idx: int ref,
+      varchar(900) tag: string ref,
+      varchar(900) tostring: string ref);
+
+case @yaml_node.kind of
+  0 = @yaml_scalar_node
+| 1 = @yaml_mapping_node
+| 2 = @yaml_sequence_node
+| 3 = @yaml_alias_node
+;
+
+@yaml_collection_node = @yaml_mapping_node | @yaml_sequence_node;
+
+@yaml_node_parent = @yaml_collection_node | @file;
+
+yaml_anchors (unique int node: @yaml_node ref,
+              varchar(900) anchor: string ref);
+
+yaml_aliases (unique int alias: @yaml_alias_node ref,
+              varchar(900) target: string ref);
+
+yaml_scalars (unique int scalar: @yaml_scalar_node ref,
+              int style: int ref,
+              varchar(900) value: string ref);
+
+yaml_errors (unique int id: @yaml_error,
+             varchar(900) message: string ref);
+
+yaml_locations(unique int locatable: @yaml_locatable ref,
+             int location: @location_default ref);
+
+@yaml_locatable = @yaml_node | @yaml_error;

python/ql/lib/upgrades/47e552c4357a04c5735355fad818630daee4a5ac/upgrade.properties

@@ -0,0 +1,2 @@
+description: Add YAML tables
+compatibility: full