Python: Move concept tests out

These tests should be fleshed out at some point, but currently
they test all that we model.

python/ql/src/experimental/semmle/python/frameworks/Dill.qll

@@ -20,6 +20,7 @@ private module Dill {
   /** Gets a reference to the `dill` module. */
   DataFlow::Node dill() { result = dill(DataFlow::TypeTracker::end()) }
 
+  /** Provides models for the `dill` module. */
   module dill {
     /** Gets a reference to the `dill.loads` function. */
     private DataFlow::Node loads(DataFlow::TypeTracker t) {
@@ -55,5 +56,10 @@ private class DillDeserialization extends UnmarshalingFunction::Range {
 
   override DataFlow::Node getOutput() { result = this }
 
-  override string getFormat() { none() }
+  override string getFormat() {
+    result = this.asCfgNode().(CallNode).getArgByName("encoding").(NameNode).getId()
+    or
+    not exists(this.asCfgNode().(CallNode).getArgByName("encoding")) and
+    result = "ASCII"
+  }
 }

python/ql/src/experimental/semmle/python/frameworks/Stdlib.qll

@@ -342,6 +342,7 @@ private module Stdlib {
   /** Gets a reference to the `marshal` module. */
   DataFlow::Node marshal() { result = marshal(DataFlow::TypeTracker::end()) }
 
+  /** Provides models for the `marshal` module. */
   module marshal {
     /** Gets a reference to the `marshal.loads` function. */
     private DataFlow::Node loads(DataFlow::TypeTracker t) {
@@ -394,6 +395,7 @@ private module Stdlib {
   /** Gets a reference to the `pickle` module. */
   DataFlow::Node pickle() { result = pickle(DataFlow::TypeTracker::end()) }
 
+  /** Provides models for the `pickle` module. */
   module pickle {
     /** Gets a reference to the `pickle.loads` function. */
     private DataFlow::Node loads(DataFlow::TypeTracker t) {
@@ -429,6 +431,9 @@ private module Stdlib {
 
     override string getFormat() {
       result = this.asCfgNode().(CallNode).getArgByName("encoding").(NameNode).getId()
+      or
+      not exists(this.asCfgNode().(CallNode).getArgByName("encoding")) and
+      result = "ASCII"
     }
   }
 }

python/ql/src/experimental/semmle/python/frameworks/Yaml.qll

@@ -20,6 +20,7 @@ private module Yaml {
   /** Gets a reference to the `yaml` module. */
   DataFlow::Node yaml() { result = yaml(DataFlow::TypeTracker::end()) }
 
+  /** Provides models for the `yaml` module. */
   module yaml {
     /** Gets a reference to the `yaml.load` function. */
     private DataFlow::Node load(DataFlow::TypeTracker t) {

python/ql/test/experimental/library-tests/frameworks/dill/UnmarshalFunction.py

@@ -0,0 +1,13 @@
+import flask
+import dill
+
+from flask import Flask, request
+
+app = Flask(__name__)
+
+
+@app.route("/")
+def hello():
+    payload = request.args.get("payload")
+    dill.loads(payload)  # $UNSAFE_getAnInput=payload $UNSAFE_getOutput=Attribute() $UNSAFE_getFormat=ASCII
+    dill.loads(payload, encoding='latin1')  # $UNSAFE_getAnInput=payload $UNSAFE_getOutput=Attribute() $UNSAFE_getFormat=latin1

python/ql/test/experimental/library-tests/frameworks/stdlib/UnmarshalFunction.py

@@ -0,0 +1,15 @@
+import flask
+import pickle
+import marshal
+
+from flask import Flask, request
+
+app = Flask(__name__)
+
+
+@app.route("/")
+def hello():
+    payload = request.args.get("payload")
+    pickle.loads(payload)  # $UNSAFE_getAnInput=payload $UNSAFE_getOutput=Attribute() $UNSAFE_getFormat=ASCII
+    pickle.loads(payload, encoding='latin1')  # $UNSAFE_getAnInput=payload $UNSAFE_getOutput=Attribute() $UNSAFE_getFormat=latin1
+    marshal.loads(payload)  # $UNSAFE_getAnInput=payload $UNSAFE_getOutput=Attribute()

python/ql/test/experimental/library-tests/frameworks/yaml/ConceptsTest.ql

@@ -0,0 +1,2 @@
+import python
+import experimental.meta.ConceptsTest

python/ql/test/experimental/library-tests/frameworks/yaml/UnmarshalFunction.py

@@ -0,0 +1,14 @@
+import flask
+import yaml
+from yaml import SafeLoader
+
+from flask import Flask, request
+
+app = Flask(__name__)
+
+
+@app.route("/")
+def hello():
+    payload = request.args.get("payload")
+    yaml.load(payload)  # $UNSAFE_getAnInput=payload $UNSAFE_getOutput=Attribute()
+    yaml.load(payload, Loader=SafeLoader)  # $getAnInput=payload $getOutput=Attribute()

python/ql/test/experimental/query-tests/Security-new-dataflow/CWE-502/unsafe_deserialization.py

@@ -12,10 +12,10 @@ app = Flask(__name__)
 @app.route("/")
 def hello():
     payload = request.args.get("payload")
-    pickle.loads(payload)  # $UNSAFE_getAnInput=payload $UNSAFE_getOutput=Attribute()
-    yaml.load(payload)  # $UNSAFE_getAnInput=payload $UNSAFE_getOutput=Attribute()
-    yaml.load(payload, Loader=SafeLoader)  # $getAnInput=payload $getOutput=Attribute()
-    marshal.loads(payload)  # $UNSAFE_getAnInput=payload $UNSAFE_getOutput=Attribute()
+    pickle.loads(payload)
+    yaml.load(payload)
+    yaml.load(payload, Loader=SafeLoader)
+    marshal.loads(payload)
 
     import dill
-    dill.loads(payload)  # $UNSAFE_getAnInput=payload $UNSAFE_getOutput=Attribute()
+    dill.loads(payload)