add test

python/ql/lib/semmle/python/frameworks/Aiohttp.qll

@@ -644,7 +644,7 @@ module AiohttpWebModel {
  * Provides models for the web server part (`aiohttp.client`) of the `aiohttp` PyPI package.
  * See https://docs.aiohttp.org/en/stable/client.html
  */
-module AiohttpClientModel {
+private module AiohttpClientModel {
   /**
    * Provides models for the `aiohttp.ClientSession` class
    *
@@ -668,7 +668,7 @@ module AiohttpClientModel {
         this = instance().getMember(methodName).getACall()
       }
 
-      DataFlow::Node getUrlArg() {
+      override DataFlow::Node getAUrlPart() {
         result = this.getArgByName("url")
         or
         not methodName = "request" and
@@ -678,13 +678,12 @@ module AiohttpClientModel {
         result = this.getArg(1)
       }
 
-      override DataFlow::Node getAUrlPart() { result = this.getUrlArg() }
-
       override string getFramework() { result = "aiohttp.ClientSession" }
 
       override predicate disablesCertificateValidation(
         DataFlow::Node disablingNode, DataFlow::Node argumentOrigin
       ) {
+        // TODO: Look into disabling certificate validation
         none()
       }
     }

python/ql/lib/semmle/python/frameworks/Httpx.qll

@@ -20,7 +20,7 @@ module HttpxModel {
       this = API::moduleImport("httpx").getMember(methodName).getACall()
     }
 
-    DataFlow::Node getUrlArg() {
+    override DataFlow::Node getAUrlPart() {
       result = this.getArgByName("url")
       or
       not methodName = "request" and
@@ -30,13 +30,12 @@ module HttpxModel {
       result = this.getArg(1)
     }
 
-    override DataFlow::Node getAUrlPart() { result = this.getUrlArg() }
-
     override string getFramework() { result = "httpx" }
 
     override predicate disablesCertificateValidation(
       DataFlow::Node disablingNode, DataFlow::Node argumentOrigin
     ) {
+      // TODO: Look into disabling certificate validation
       none()
     }
   }
@@ -64,7 +63,7 @@ module HttpxModel {
         this = instance().getMember(methodName).getACall()
       }
 
-      DataFlow::Node getUrlArg() {
+      override DataFlow::Node getAUrlPart() {
         result = this.getArgByName("url")
         or
         not methodName = "request" and
@@ -74,13 +73,12 @@ module HttpxModel {
         result = this.getArg(1)
       }
 
-      override DataFlow::Node getAUrlPart() { result = this.getUrlArg() }
-
       override string getFramework() { result = "httpx.[Async]Client" }
 
       override predicate disablesCertificateValidation(
         DataFlow::Node disablingNode, DataFlow::Node argumentOrigin
       ) {
+        // TODO: Look into disabling certificate validation
         none()
       }
     }

python/ql/lib/semmle/python/frameworks/Libtaxii.qll

@@ -22,15 +22,14 @@ module Libtaxii {
       this.getArgByName("allow_url").asExpr().toString() = "True"
     }
 
-    DataFlow::Node getUrlArg() { result in [this.getArg(0), this.getArgByName("s")] }
-
-    override DataFlow::Node getAUrlPart() { result = this.getUrlArg() }
+    override DataFlow::Node getAUrlPart() { result in [this.getArg(0), this.getArgByName("s")] }
 
     override string getFramework() { result = "libtaxii.common.parse" }
 
     override predicate disablesCertificateValidation(
       DataFlow::Node disablingNode, DataFlow::Node argumentOrigin
     ) {
+      // TODO: Look into disabling certificate validation
       none()
     }
   }

python/ql/lib/semmle/python/frameworks/Pycurl.qll

@@ -25,7 +25,7 @@ module Pycurl {
     private API::Node instance() { result = classRef().getReturn() }
 
     /**
-     * When the first parameter value of the `setopt` function is set to `pycurl.URL`, 
+     * When the first parameter value of the `setopt` function is set to `pycurl.URL`,
      * the second parameter value is the request resource link.
      *
      * See https://pycurl.io/docs/latest/curl.html#set_option.
@@ -36,15 +36,16 @@ module Pycurl {
         this.getArg(0).asCfgNode().(AttrNode).getName() = "URL"
       }
 
-      DataFlow::Node getUrlArg() { result in [this.getArg(1), this.getArgByName("value")] }
-
-      override DataFlow::Node getAUrlPart() { result = this.getUrlArg() }
+      override DataFlow::Node getAUrlPart() {
+        result in [this.getArg(1), this.getArgByName("value")]
+      }
 
       override string getFramework() { result = "pycurl.Curl" }
 
       override predicate disablesCertificateValidation(
         DataFlow::Node disablingNode, DataFlow::Node argumentOrigin
       ) {
+        // TODO: Look into disabling certificate validation
         none()
       }
     }

python/ql/lib/semmle/python/frameworks/Urllib.qll

@@ -27,15 +27,14 @@ module Urllib {
         this = API::moduleImport("urllib").getMember("request").getMember("Request").getACall()
       }
 
-      DataFlow::Node getUrlArg() { result in [this.getArg(0), this.getArgByName("url")] }
-
-      override DataFlow::Node getAUrlPart() { result = this.getUrlArg() }
+      override DataFlow::Node getAUrlPart() { result in [this.getArg(0), this.getArgByName("url")] }
 
       override string getFramework() { result = "urllib.request.Request" }
 
       override predicate disablesCertificateValidation(
         DataFlow::Node disablingNode, DataFlow::Node argumentOrigin
       ) {
+        // TODO: Look into disabling certificate validation
         none()
       }
     }
@@ -49,15 +48,14 @@ module Urllib {
         this = API::moduleImport("urllib").getMember("request").getMember("urlopen").getACall()
       }
 
-      DataFlow::Node getUrlArg() { result in [this.getArg(0), this.getArgByName("url")] }
-
-      override DataFlow::Node getAUrlPart() { result = this.getUrlArg() }
+      override DataFlow::Node getAUrlPart() { result in [this.getArg(0), this.getArgByName("url")] }
 
       override string getFramework() { result = "urllib.request.urlopen" }
 
       override predicate disablesCertificateValidation(
         DataFlow::Node disablingNode, DataFlow::Node argumentOrigin
       ) {
+        // TODO: Look into disabling certificate validation
         none()
       }
     }

python/ql/lib/semmle/python/frameworks/Urllib2.qll

@@ -17,19 +17,16 @@ module Urllib2 {
    * - https://docs.python.org/2/library/urllib2.html#urllib2.Request
    */
   private class RequestCall extends HTTP::Client::Request::Range, DataFlow::CallCfgNode {
-    RequestCall() {
-      this = API::moduleImport("urllib2").getMember("Request").getACall()
-    }
+    RequestCall() { this = API::moduleImport("urllib2").getMember("Request").getACall() }
 
-    DataFlow::Node getUrlArg() { result in [this.getArg(0), this.getArgByName("url")] }
-
-    override DataFlow::Node getAUrlPart() { result = this.getUrlArg() }
+    override DataFlow::Node getAUrlPart() { result in [this.getArg(0), this.getArgByName("url")] }
 
     override string getFramework() { result = "urllib2.Request" }
 
     override predicate disablesCertificateValidation(
       DataFlow::Node disablingNode, DataFlow::Node argumentOrigin
     ) {
+      // TODO: Look into disabling certificate validation
       none()
     }
   }
@@ -41,15 +38,14 @@ module Urllib2 {
   private class UrlOpenCall extends HTTP::Client::Request::Range, DataFlow::CallCfgNode {
     UrlOpenCall() { this = API::moduleImport("urllib2").getMember("urlopen").getACall() }
 
-    DataFlow::Node getUrlArg() { result in [this.getArg(0), this.getArgByName("url")] }
-
-    override DataFlow::Node getAUrlPart() { result = this.getUrlArg() }
+    override DataFlow::Node getAUrlPart() { result in [this.getArg(0), this.getArgByName("url")] }
 
     override string getFramework() { result = "urllib2.urlopen" }
 
     override predicate disablesCertificateValidation(
       DataFlow::Node disablingNode, DataFlow::Node argumentOrigin
     ) {
+      // TODO: Look into disabling certificate validation
       none()
     }
   }

python/ql/lib/semmle/python/frameworks/Urllib3.qll

@@ -30,15 +30,14 @@ module Urllib3 {
           instance().getMember(["request", "request_encode_url", "request_encode_body"]).getACall()
       }
 
-      DataFlow::Node getUrlArg() { result in [this.getArg(1), this.getArgByName("url")] }
-
-      override DataFlow::Node getAUrlPart() { result = this.getUrlArg() }
+      override DataFlow::Node getAUrlPart() { result in [this.getArg(1), this.getArgByName("url")] }
 
       override string getFramework() { result = "urllib3.PoolManager" }
 
       override predicate disablesCertificateValidation(
         DataFlow::Node disablingNode, DataFlow::Node argumentOrigin
       ) {
+        // TODO: Look into disabling certificate validation
         none()
       }
     }
@@ -46,15 +45,14 @@ module Urllib3 {
     private class UrlOpenCall extends HTTP::Client::Request::Range, DataFlow::CallCfgNode {
       UrlOpenCall() { this = instance().getMember("urlopen").getACall() }
 
-      DataFlow::Node getUrlArg() { result in [this.getArg(1), this.getArgByName("url")] }
-
-      override DataFlow::Node getAUrlPart() { result = this.getUrlArg() }
+      override DataFlow::Node getAUrlPart() { result in [this.getArg(1), this.getArgByName("url")] }
 
       override string getFramework() { result = "urllib3.PoolManager" }
 
       override predicate disablesCertificateValidation(
         DataFlow::Node disablingNode, DataFlow::Node argumentOrigin
       ) {
+        // TODO: Look into disabling certificate validation
         none()
       }
     }

python/ql/test/library-tests/frameworks/aiohttp/client_request.py

@@ -0,0 +1,16 @@
+import aiohttp
+import asyncio
+
+s = aiohttp.ClientSession()
+resp = s.request("method", "url") # $ clientRequestUrlPart="url"
+resp = s.request("method", url="url") # $ clientRequestUrlPart="url"
+
+with aiohttp.ClientSession() as session:
+    resp = session.get("url") # $ clientRequestUrlPart="url"
+    resp = session.request(method="GET", url="url") # $ clientRequestUrlPart="url"
+
+# other methods than GET
+s = aiohttp.ClientSession()
+resp = s.post("url") # $ clientRequestUrlPart="url"
+resp = s.patch("url") # $ clientRequestUrlPart="url"
+resp = s.options("url") # $ clientRequestUrlPart="url"

python/ql/test/library-tests/frameworks/httpx/ConceptsTest.ql

@@ -0,0 +1,2 @@
+import python
+import experimental.meta.ConceptsTest

python/ql/test/library-tests/frameworks/httpx/test.py

@@ -0,0 +1,24 @@
+import httpx
+
+httpx.get("url") # $ clientRequestUrlPart="url"
+httpx.post("url") # $ clientRequestUrlPart="url"
+httpx.patch("url") # $ clientRequestUrlPart="url"
+httpx.options("url") # $ clientRequestUrlPart="url"
+httpx.request("method", url="url") # $ clientRequestUrlPart="url"
+httpx.stream("method", url="url") # $ clientRequestUrlPart="url"
+
+client = httpx.Client()
+response = client.get("url") # $ clientRequestUrlPart="url"
+response = client.post("url") # $ clientRequestUrlPart="url"
+response = client.patch("url") # $ clientRequestUrlPart="url"
+response = client.options("url") # $ clientRequestUrlPart="url"
+response = client.request("method", url="url") # $ clientRequestUrlPart="url"
+response = client.stream("method", url="url") # $ clientRequestUrlPart="url"
+
+client = httpx.AsyncClient()
+response = client.get("url") # $ clientRequestUrlPart="url"
+response = client.post("url") # $ clientRequestUrlPart="url"
+response = client.patch("url") # $ clientRequestUrlPart="url"
+response = client.options("url") # $ clientRequestUrlPart="url"
+response = client.request("method", url="url") # $ clientRequestUrlPart="url"
+response = client.stream("method", url="url") # $ clientRequestUrlPart="url"

python/ql/test/library-tests/frameworks/libtaxii/ConceptsTest.ql

@@ -0,0 +1,2 @@
+import python
+import experimental.meta.ConceptsTest

python/ql/test/library-tests/frameworks/libtaxii/test.py

@@ -0,0 +1,4 @@
+from libtaxii.common import parse
+
+result = parse("url", allow_url=True) # $ clientRequestUrlPart="url"
+result = parse(s="url", allow_url=True) # $ clientRequestUrlPart="url"

python/ql/test/library-tests/frameworks/pycurl/ConceptsTest.ql

@@ -0,0 +1,2 @@
+import python
+import experimental.meta.ConceptsTest

python/ql/test/library-tests/frameworks/pycurl/test.py

@@ -0,0 +1,4 @@
+import pycurl
+
+c = pycurl.Curl()
+c.setopt(pycurl.URL, "url") # $ clientRequestUrlPart="url"

python/ql/test/library-tests/frameworks/urllib/ConceptsTest.ql

@@ -0,0 +1,2 @@
+import python
+import experimental.meta.ConceptsTest

python/ql/test/library-tests/frameworks/urllib/test.py

@@ -0,0 +1,7 @@
+from urllib.request import Request, urlopen
+
+Request("url") # $ clientRequestUrlPart="url"
+Request(url="url") # $ clientRequestUrlPart="url"
+
+urlopen("url") # $ clientRequestUrlPart="url"
+urlopen(url="url") # $ clientRequestUrlPart="url"

python/ql/test/library-tests/frameworks/urllib2/ConceptsTest.ql

@@ -0,0 +1,2 @@
+import python
+import experimental.meta.ConceptsTest

python/ql/test/library-tests/frameworks/urllib2/test.py

@@ -0,0 +1,7 @@
+import urllib2
+
+resp = urllib2.Request("url") # $ clientRequestUrlPart="url"
+resp = urllib2.Request(url="url") # $ clientRequestUrlPart="url"
+
+resp = urllib2.urlopen("url") # $ clientRequestUrlPart="url"
+resp = urllib2.urlopen(url="url") # $ clientRequestUrlPart="url"

python/ql/test/library-tests/frameworks/urllib3/ConceptsTest.ql

@@ -0,0 +1,2 @@
+import python
+import experimental.meta.ConceptsTest

python/ql/test/library-tests/frameworks/urllib3/test.py

@@ -0,0 +1,8 @@
+import urllib3
+
+http = urllib3.PoolManager()
+
+resp = http.request("method", "url") # $ clientRequestUrlPart="url"
+resp = http.request("method", url="url") # $ clientRequestUrlPart="url"
+resp = http.urlopen("method", "url") # $ clientRequestUrlPart="url"
+resp = http.urlopen("method", url="url") # $ clientRequestUrlPart="url"