Merge branch 'main' of https://github.com/github/codeql into python/add-comprehension-capture-flow

python/ql/lib/CHANGELOG.md

@@ -1,3 +1,13 @@
+## 2.1.0
+
+### New Features
+
+* Added support for custom threat-models, which can be used in most of our taint-tracking queries, see our [documentation](https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning#extending-codeql-coverage-with-threat-models) for more details.
+
+### Minor Analysis Improvements
+
+* The common sanitizer guard `StringConstCompareBarrier` has been renamed to `ConstCompareBarrier` and expanded to cover comparisons with other constant values such as `None`. This may result in fewer false positive results for several queries. 
+
 ## 2.0.0
 
 ### Breaking Changes

python/ql/lib/change-notes/2024-09-20-const-compare-gaurd.md

@@ -1,4 +0,0 @@
----
-category: minorAnalysis
----
-* The common sanitizer guard `StringConstCompareBarrier` has been renamed to `ConstCompareBarrier` and expanded to cover comparisons with other constant values such as `None`. This may result in fewer false positive results for several queries. 

python/ql/lib/change-notes/2024-09-24-std-lib-models.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Added several models of standard library functions and classes, in anticipation of no longer extracting the standard library in a future release.

python/ql/lib/change-notes/released/2.1.0.md

@@ -0,0 +1,9 @@
+## 2.1.0
+
+### New Features
+
+* Added support for custom threat-models, which can be used in most of our taint-tracking queries, see our [documentation](https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning#extending-codeql-coverage-with-threat-models) for more details.
+
+### Minor Analysis Improvements
+
+* The common sanitizer guard `StringConstCompareBarrier` has been renamed to `ConstCompareBarrier` and expanded to cover comparisons with other constant values such as `None`. This may result in fewer false positive results for several queries. 

python/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 2.0.0
+lastReleaseVersion: 2.1.0

python/ql/lib/ext/default-threat-models-fixup.model.yml

@@ -0,0 +1,8 @@
+extensions:
+  - addsTo:
+      pack: codeql/threat-models
+      extensible: threatModelConfiguration
+    data:
+    # Since responses are enabled by default in the shared threat-models configuration,
+    # we need to disable it here to keep existing behavior for the python analysis.
+      - ["response", false, -2147483647]

python/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/python-all
-version: 2.0.1-dev
+version: 2.1.1-dev
 groups: python
 dbscheme: semmlecode.python.dbscheme
 extractor: python
@@ -9,10 +9,12 @@ dependencies:
   codeql/dataflow: ${workspace}
   codeql/mad: ${workspace}
   codeql/regex: ${workspace}
+  codeql/threat-models: ${workspace}
   codeql/tutorial: ${workspace}
   codeql/util: ${workspace}
   codeql/xml: ${workspace}
   codeql/yaml: ${workspace}
 dataExtensions:
   - semmle/python/frameworks/**/*.model.yml
+  - ext/*.model.yml
 warnOnImplicitThis: true

python/ql/lib/semmle/python/Concepts.qll

@@ -10,6 +10,62 @@ private import semmle.python.dataflow.new.RemoteFlowSources
 private import semmle.python.dataflow.new.TaintTracking
 private import semmle.python.Frameworks
 private import semmle.python.security.internal.EncryptionKeySizes
+private import codeql.threatmodels.ThreatModels
+
+/**
+ * A data flow source, for a specific threat-model.
+ *
+ * Extend this class to refine existing API models. If you want to model new APIs,
+ * extend `ThreatModelSource::Range` instead.
+ */
+class ThreatModelSource extends DataFlow::Node instanceof ThreatModelSource::Range {
+  /**
+   * Gets a string that represents the source kind with respect to threat modeling.
+   *
+   * See
+   * - https://github.com/github/codeql/blob/main/docs/codeql/reusables/threat-model-description.rst
+   * - https://github.com/github/codeql/blob/main/shared/threat-models/ext/threat-model-grouping.model.yml
+   */
+  string getThreatModel() { result = super.getThreatModel() }
+
+  /** Gets a string that describes the type of this threat-model source. */
+  string getSourceType() { result = super.getSourceType() }
+}
+
+/** Provides a class for modeling new sources for specific threat-models. */
+module ThreatModelSource {
+  /**
+   * A data flow source, for a specific threat-model.
+   *
+   * Extend this class to model new APIs. If you want to refine existing API models,
+   * extend `ThreatModelSource` instead.
+   */
+  abstract class Range extends DataFlow::Node {
+    /**
+     * Gets a string that represents the source kind with respect to threat modeling.
+     *
+     * See
+     * - https://github.com/github/codeql/blob/main/docs/codeql/reusables/threat-model-description.rst
+     * - https://github.com/github/codeql/blob/main/shared/threat-models/ext/threat-model-grouping.model.yml
+     */
+    abstract string getThreatModel();
+
+    /** Gets a string that describes the type of this threat-model source. */
+    abstract string getSourceType();
+  }
+}
+
+/**
+ * A data flow source that is enabled in the current threat model configuration.
+ */
+class ActiveThreatModelSource extends ThreatModelSource {
+  ActiveThreatModelSource() {
+    exists(string kind |
+      currentThreatModel(kind) and
+      this.getThreatModel() = kind
+    )
+  }
+}
 
 /**
  * A data-flow node that executes an operating system command,

python/ql/lib/semmle/python/dataflow/new/RemoteFlowSources.qll

@@ -15,10 +15,7 @@ private import semmle.python.Concepts
  * Extend this class to refine existing API models. If you want to model new APIs,
  * extend `RemoteFlowSource::Range` instead.
  */
-class RemoteFlowSource extends DataFlow::Node instanceof RemoteFlowSource::Range {
-  /** Gets a string that describes the type of this remote flow source. */
-  string getSourceType() { result = super.getSourceType() }
-}
+class RemoteFlowSource extends ThreatModelSource instanceof RemoteFlowSource::Range { }
 
 /** Provides a class for modeling new sources of remote user input. */
 module RemoteFlowSource {
@@ -28,8 +25,7 @@ module RemoteFlowSource {
    * Extend this class to model new APIs. If you want to refine existing API models,
    * extend `RemoteFlowSource` instead.
    */
-  abstract class Range extends DataFlow::Node {
-    /** Gets a string that describes the type of this remote flow source. */
-    abstract string getSourceType();
+  abstract class Range extends ThreatModelSource::Range {
+    override string getThreatModel() { result = "remote" }
   }
 }

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowDispatch.qll

@@ -371,14 +371,13 @@ abstract class DataFlowFunction extends DataFlowCallable, TFunction {
   int positionalOffset() { result = 0 }
 
   override ParameterNode getParameter(ParameterPosition ppos) {
+    // Do not handle lower bound positions (such as `[1..]`) here
+    // they are handled by parameter matching and would create
+    // inconsistencies here as multiple parameters could match such a position.
     exists(int index | ppos.isPositional(index) |
       result.getParameter() = func.getArg(index + this.positionalOffset())
     )
     or
-    exists(int index1, int index2 | ppos.isPositionalLowerBound(index1) and index2 >= index1 |
-      result.getParameter() = func.getArg(index2 + this.positionalOffset())
-    )
-    or
     exists(string name | ppos.isKeyword(name) | result.getParameter() = func.getArgByName(name))
     or
     // `*args`

python/ql/lib/semmle/python/dataflow/new/internal/TaintTrackingPrivate.qll

@@ -46,8 +46,6 @@ private module Cached {
       or
       containerStep(nodeFrom, nodeTo)
       or
-      copyStep(nodeFrom, nodeTo)
-      or
       DataFlowPrivate::forReadStep(nodeFrom, _, nodeTo)
       or
       DataFlowPrivate::iterableUnpackingReadStep(nodeFrom, _, nodeTo)
@@ -191,18 +189,6 @@ predicate containerStep(DataFlow::Node nodeFrom, DataFlow::Node nodeTo) {
   DataFlowPrivate::yieldStoreStep(nodeFrom, _, nodeTo)
 }
 
-/**
- * Holds if taint can flow from `nodeFrom` to `nodeTo` with a step related to copying.
- */
-predicate copyStep(DataFlow::CfgNode nodeFrom, DataFlow::CfgNode nodeTo) {
-  exists(DataFlow::CallCfgNode call | call = nodeTo |
-    call = API::moduleImport("copy").getMember(["copy", "deepcopy"]).getACall() and
-    call.getArg(0) = nodeFrom
-  )
-  or
-  nodeTo.(DataFlow::MethodCallNode).calls(nodeFrom, "copy")
-}
-
 /**
  * Holds if taint can flow from `nodeFrom` to `nodeTo` with an `await`-step,
  * such that the whole expression `await x` is tainted if `x` is tainted.

python/ql/lib/semmle/python/frameworks/PEP249.qll

@@ -81,6 +81,24 @@ module PEP249 {
     }
   }
 
+  /** A call to a method that fetches rows from a previous execution. */
+  private class FetchMethodCall extends ThreatModelSource::Range, API::CallNode {
+    FetchMethodCall() {
+      exists(API::Node start |
+        start instanceof DatabaseCursor or start instanceof DatabaseConnection
+      |
+        // note: since we can't currently provide accesspaths for sources, these are all
+        // lumped together, although clearly the fetchmany/fetchall returns a
+        // list/iterable with rows.
+        this = start.getMember(["fetchone", "fetchmany", "fetchall"]).getACall()
+      )
+    }
+
+    override string getThreatModel() { result = "database" }
+
+    override string getSourceType() { result = "cursor.fetch*()" }
+  }
+
   // ---------------------------------------------------------------------------
   // asyncio implementations
   // ---------------------------------------------------------------------------

python/ql/lib/semmle/python/frameworks/Stdlib.model.yml

@@ -0,0 +1,183 @@
+extensions:
+  - addsTo:
+      pack: codeql/python-all
+      extensible: sourceModel
+    data:
+      - ['os', 'Member[getenv].ReturnValue', 'environment']
+      - ['os', 'Member[getenvb].ReturnValue', 'environment']
+      - ['os', 'Member[environ]', 'environment']
+      - ['os', 'Member[environb]', 'environment']
+      - ['posix', 'Member[environ]', 'environment']
+
+      - ['sys', 'Member[argv]', 'commandargs']
+      - ['sys', 'Member[orig_argv]', 'commandargs']
+
+      - ['sys', 'Member[stdin]', 'stdin']
+      - ['builtins', 'Member[input].ReturnValue', 'stdin']
+      - ['builtins', 'Member[raw_input].ReturnValue', 'stdin'] # python 2 only
+
+
+      # if no argument is given, the default is to use sys.argv[1:]
+      - ['argparse.ArgumentParser', 'Member[parse_args,parse_known_args].WithArity[0].ReturnValue', 'commandargs']
+
+      - ['os', 'Member[read].ReturnValue', 'file']
+
+  - addsTo:
+      pack: codeql/python-all
+      extensible: sinkModel
+    data:
+      - ["zipfile.ZipFile","Member[extractall].Argument[0,path:]", "path-injection"]
+
+  - addsTo:
+      pack: codeql/python-all
+      extensible: summaryModel
+    data:
+      # See https://docs.python.org/3/library/argparse.html#argparse.ArgumentParser
+      # note: taint flow for attribute lookups on `argparse.ArgumentParser` is handled in QL
+      - ["argparse.ArgumentParser", "Member[_parse_known_args,_read_args_from_files]", "Argument[0,arg_strings:]", "ReturnValue", "taint"]
+      - ["argparse.ArgumentParser", "Member[parse_args,parse_known_args]", "Argument[0,args:]", "ReturnValue", "taint"]
+      # See https://docs.python.org/3/library/cgi.html#higher-level-interface
+      - ["cgi.FieldStorage", "Member[getfirst,getlist,getvalue]", "Argument[self]", "ReturnValue", "taint"]
+      # See
+      #  - https://docs.python.org/3/glossary.html#term-mapping 
+      #  - https://docs.python.org/3/library/stdtypes.html#dict.get
+      - ["collections.abc.Mapping", "Member[get]", "Argument[1,default:]", "ReturnValue", "taint"]
+      # See https://docs.python.org/3/library/contextlib.html#contextlib.ExitStack
+      - ["contextlib.ExitStack", "Member[enter_context]", "Argument[0,cm:]", "ReturnValue", "taint"]
+      # See https://docs.python.org/3/library/copy.html#copy.deepcopy
+      - ["copy", "Member[copy,deepcopy]", "Argument[0,x:]", "ReturnValue", "value"]
+      # See
+      #  - https://docs.python.org/3/library/ctypes.html#ctypes.create_string_buffer
+      #  - https://docs.python.org/3/library/ctypes.html#ctypes.create_unicode_buffer
+      - ["ctypes", "Member[create_string_buffer,create_unicode_buffer]", "Argument[0,init:,init_or_size:]", "ReturnValue", "taint"]
+      # See https://docs.python.org/3.11/distutils/apiref.html#distutils.util.change_root
+      - ["distutils", "Member[util].Member[change_root]", "Argument[0,new_root:,1,pathname:]", "ReturnValue", "taint"]
+      # See https://docs.python.org/3/library/email.header.html#email.header.Header
+      - ["email.header.Header!", "Subclass.Call", "Argument[0,s:]", "ReturnValue", "taint"]
+      # See https://docs.python.org/3/library/email.utils.html#email.utils.parseaddr
+      - ["email", "Member[utils].Member[parseaddr]", "Argument[0,addr:]", "ReturnValue", "taint"]
+      - ["email", "Member[utils].Member[parseaddr]", "Argument[0,addr:]", "ReturnValue.TupleElement[0,1]", "taint"]
+      # See See https://docs.python.org/3/library/fnmatch.html#fnmatch.filter
+      - ["fnmatch", "Member[filter]", "Argument[0,names:].ListElement", "ReturnValue.ListElement", "value"]
+      - ["fnmatch", "Member[filter]", "Argument[0,names:]", "ReturnValue", "taint"]
+      # See https://docs.python.org/3/library/getopt.html#getopt.getopt
+      - ["getopt", "Member[getopt]", "Argument[0,args:]", "ReturnValue.TupleElement[1]", "taint"]
+      - ["getopt", "Member[getopt]", "Argument[1,shortopts:,2,longopts:]", "ReturnValue.TupleElement[0].ListElement.TupleElement[0]", "taint"]
+      # See https://docs.python.org/3/library/gettext.html#gettext.gettext
+      - ["gettext", "Member[gettext]", "Argument[0,message:]", "ReturnValue", "taint"]
+      # See https://docs.python.org/3/library/gzip.html#gzip.GzipFile
+      - ["gzip.GzipFile!", "Subclass.Call", "Argument[0,filename:]", "ReturnValue", "taint"]
+      # See
+      #  - https://docs.python.org/3/library/html.html#html.escape
+      #  - https://docs.python.org/3/library/html.html#html.unescape
+      - ["html", "Member[escape,unescape]", "Argument[0,s:]", "ReturnValue", "taint"]
+      # See https://docs.python.org/3/library/html.parser.html#html.parser.HTMLParser.feed
+      - ["html.parser.HTMLParser", "Member[feed]", "Argument[0,data:]", "Argument[self]", "taint"]
+      # See https://docs.python.org/3.11/library/imp.html#imp.find_module
+      - ["imp", "Member[find_module]", "Argument[0,name:,1,path:]", "ReturnValue", "taint"]
+      # See https://docs.python.org/3/library/logging.html#logging.getLevelName
+      #   specifically the no matching case 
+      - ["logging", "Member[getLevelName]", "Argument[0,level:]", "ReturnValue", "taint"]
+      # See https://docs.python.org/3/library/logging.html#logging.LogRecord.getMessage
+      - ["logging.LogRecord", "Member[getMessage]", "Argument[self]", "ReturnValue", "taint"]
+      # See https://docs.python.org/3/library/mimetypes.html#mimetypes.guess_type
+      - ["mimetypes", "Member[guess_type]", "Argument[0,url:]", "ReturnValue", "taint"]
+      # See https://github.com/python/cpython/blob/main/Lib/nturl2path.py
+      #   No user-facing documentation, unfortunately.
+      - ["nturl2path", "Member[pathname2url]", "Argument[0,p:]", "ReturnValue", "taint"]
+      - ["nturl2path", "Member[url2pathname]", "Argument[0,url:]", "ReturnValue", "taint"]
+      # See https://docs.python.org/3/library/optparse.html#optparse.OptionParser.parse_args
+      - ["optparse.OptionParser", "Member[parse_args]", "Argument[0,args:,1,values:]", "ReturnValue.TupleElement[0,1]", "taint"]
+      # See https://github.com/python/cpython/blob/3.10/Lib/pathlib.py#L972-L973
+      - ["pathlib.Path", ".Member[__enter__]", "Argument[self]", "ReturnValue", "taint"]
+      # See https://docs.python.org/3/library/os.html#os.PathLike.__fspath__
+      - ["pathlib.PurePath", "Member[__fspath__]", "Argument[self]", "ReturnValue", "taint"]
+      # See
+      #  - https://docs.python.org/3/library/asyncio-queue.html#asyncio.Queue.get
+      #  - https://docs.python.org/3/library/asyncio-queue.html#asyncio.Queue.get_nowait
+      - ["queue.Queue", "Member[get,get_nowait]", "Argument[self].ListElement", "ReturnValue", "value"]
+      - ["queue.Queue", "Member[get,get_nowait]", "Argument[self]", "ReturnValue", "taint"]
+      # See
+      #  - https://docs.python.org/3/library/asyncio-queue.html#asyncio.Queue.put
+      #  - https://docs.python.org/3/library/asyncio-queue.html#asyncio.Queue.put_nowait
+      - ["queue.Queue", "Member[put,put_nowait]", "Argument[0,item:]", "Argument[self].ListElement", "value"]
+      - ["queue.Queue", "Member[put,put_nowait]", "Argument[0,item:]", "Argument[self]", "taint"]
+      # See 
+      #  - https://docs.python.org/3/library/random.html#random.choice
+      #  - https://docs.python.org/3/library/random.html#module-random
+      - ["random", "Member[choice]", "Argument[0,seq:].ListElement", "ReturnValue", "value"]
+      - ["random", "Member[choice]", "Argument[0,seq:].SetElement", "ReturnValue", "value"]
+      - ["random", "Member[choice]", "Argument[0,seq:]", "ReturnValue", "taint"]
+      - ["random.Random", "Member[choice]", "Argument[0,seq:].ListElement", "ReturnValue", "value"]
+      - ["random.Random", "Member[choice]", "Argument[0,seq:].SetElement", "ReturnValue", "value"]
+      - ["random.Random", "Member[choice]", "Argument[0,seq:]", "ReturnValue", "taint"]
+      # See https://docs.python.org/3/library/shlex.html#shlex.quote
+      - ["shlex", "Member[quote]", "Argument[0,s:]", "ReturnValue", "taint"]
+      # See https://docs.python.org/3/library/shutil.html#shutil.rmtree
+      - ["shutil", "Member[rmtree]", "Argument[0,path:]", "Argument[2,onerror:,onexc:].Parameter[1]", "taint"]
+      # See https://docs.python.org/3/library/shutil.html#shutil.which
+      - ["shutil", "Member[which]", "Argument[0,cmd:,2,path:]", "ReturnValue", "taint"]
+      # See https://docs.python.org/3/library/subprocess.html#subprocess.Popen
+      - ["subprocess.Popen!", "Subclass.Call", "Argument[0,args:]", "ReturnValue", "taint"]
+      # See
+      #  - https://docs.python.org/3/library/tarfile.html#tarfile.open
+      #  - https://docs.python.org/3/library/tarfile.html#tarfile.TarFile.open
+      - ["tarfile", "Member[open]", "Argument[0,name:,2,fileobj:]", "ReturnValue", "taint"]
+      - ["tarfile.TarFile", "Member[open]", "Argument[0,name:,2,fileobj:]", "ReturnValue", "taint"]
+      # See https://docs.python.org/3/library/tempfile.html#tempfile.mkdtemp
+      - ["tempfile", "Member[mkdtemp]", "Argument[0,suffix:,1,prefix:,2,dir:]", "ReturnValue", "taint"]
+      # See https://docs.python.org/3/library/tempfile.html#tempfile.mkstemp
+      - ["tempfile", "Member[mkstemp]", "Argument[0,suffix:,1,prefix:,2,dir:]", "ReturnValue.TupleElement[0,1]", "taint"]
+      # See https://docs.python.org/3/library/textwrap.html#textwrap.dedent
+      - ["textwrap", "Member[dedent]", "Argument[0,text:]", "ReturnValue", "taint"]
+      # See https://docs.python.org/3/library/traceback.html#traceback.StackSummary.from_list
+      - ["traceback.StackSummary", "Member[from_list]", "Argument[0,a_list:]", "ReturnValue", "taint"]
+      # See https://docs.python.org/3/library/typing.html#typing.cast
+      - ["typing", "Member[cast]", "Argument[1,val:]", "ReturnValue", "value"]
+      # See https://docs.python.org/3/library/urllib.parse.html#urllib.parse.parse_qs
+      - ["urllib", "Member[parse].Member[parse_qs]", "Argument[0,qs:]", "ReturnValue", "taint"]
+      # See https://docs.python.org/3/library/urllib.parse.html#urllib.parse.quote
+      - ["urllib", "Member[parse].Member[quote]", "Argument[0,string:]", "ReturnValue", "taint"]
+      # See https://docs.python.org/3/library/urllib.parse.html#urllib.parse.quote_plus
+      - ["urllib", "Member[parse].Member[quote_plus]", "Argument[0,string:]", "ReturnValue", "taint"]
+      # See https://epydoc.sourceforge.net/stdlib/urllib-module.html
+      - ["urllib", "Member[parse].Member[splitquery]", "Argument[0,url:]", "ReturnValue.TupleElement[0,1]", "taint"]
+      # See https://docs.python.org/3/library/urllib.parse.html#urllib.parse.unquote
+      - ["urllib", "Member[parse].Member[unquote]", "Argument[0,string:]", "ReturnValue", "taint"]
+      # See https://docs.python.org/3/library/urllib.parse.html#urllib.parse.unquote_plus
+      - ["urllib", "Member[parse].Member[unquote_plus]", "Argument[0,string:]", "ReturnValue", "taint"]
+      # We could consider a more precise source than the first argument, namely tuple or dict content.
+      # See https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlencode
+      - ["urllib", "Member[parse].Member[urlencode]", "Argument[0,query:]", "ReturnValue", "taint"]
+      # See https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urljoin
+      - ["urllib", "Member[parse].Member[urljoin]", "Argument[0,base:,1,url:]", "ReturnValue", "taint"]
+      # See the internal documentation
+      #   https://github.com/python/cpython/blob/3.12/Lib/zipfile/_path/__init__.py#L103-L105
+      - ["zipfile.CompleteDirs", "Member[namelist]", "Argument[self]", "ReturnValue", "taint"]
+      # See https://docs.python.org/3/library/zipfile.html#zipfile.ZipFile
+      #   it may be necessary to read the code to understand the taint propagation
+      #   Constructor: https://github.com/python/cpython/blob/3.12/Lib/zipfile/__init__.py#L1266
+      - ["zipfile.ZipFile!", "Subclass.Call", "Argument[0,file:]", "ReturnValue", "taint"]
+      - ["zipfile.ZipFile!", "Subclass.Call", "Argument[0,file:]", "ReturnValue.Attribute[filelist].ListElement.Attribute[filename]", "value"]
+      #   _extract_member: https://github.com/python/cpython/blob/3.12/Lib/zipfile/__init__.py#L1761
+      - ["zipfile.ZipFile", "Member[_extract_member]", "Argument[1,targetpath:]", "ReturnValue", "taint"]
+      #   infolist: https://github.com/python/cpython/blob/3.12/Lib/zipfile/__init__.py#L1498-L1501
+      - ["zipfile.ZipFile", "Member[infolist]", "Argument[self]", "ReturnValue", "taint"]
+      - ["zipfile.ZipFile", "Member[infolist]", "Argument[self].Attribute[filelist]", "ReturnValue", "value"]
+      #   namelist: https://github.com/python/cpython/blob/3.12/Lib/zipfile/__init__.py#L1494-L1496
+      - ["zipfile.ZipFile", "Member[namelist]", "Argument[self]", "ReturnValue", "taint"]
+
+  - addsTo:
+      pack: codeql/python-all
+      extensible: neutralModel
+    data: []
+
+  - addsTo:
+      pack: codeql/python-all
+      extensible: typeModel
+    data: []
+
+  - addsTo:
+      pack: codeql/python-all
+      extensible: typeVariableModel
+    data: []

python/ql/lib/semmle/python/frameworks/Stdlib.qll

@@ -254,10 +254,14 @@ module Stdlib {
    * See https://docs.python.org/3.9/library/logging.html#logging.Logger.
    */
   module Logger {
+    private import semmle.python.dataflow.new.internal.DataFlowDispatch as DD
+
     /** Gets a reference to the `logging.Logger` class or any subclass. */
     API::Node subclassRef() {
       result = API::moduleImport("logging").getMember("Logger").getASubclass*()
       or
+      result = API::moduleImport("logging").getMember("getLoggerClass").getReturn().getASubclass*()
+      or
       result = ModelOutput::getATypeNode("logging.Logger~Subclass").getASubclass*()
     }
 
@@ -277,6 +281,13 @@ module Stdlib {
       ClassInstantiation() {
         this = subclassRef().getACall()
         or
+        this =
+          DD::selfTracker(subclassRef()
+                .getAValueReachableFromSource()
+                .asExpr()
+                .(ClassExpr)
+                .getInnerScope())
+        or
         this = API::moduleImport("logging").getMember("root").asSource()
         or
         this = API::moduleImport("logging").getMember("getLogger").getACall()
@@ -338,7 +349,7 @@ module StdlibPrivate {
    * Modeling of path related functions in the `os` module.
    * Wrapped in QL module to make it easy to fold/unfold.
    */
-  private module OsFileSystemAccessModeling {
+  module OsFileSystemAccessModeling {
     /**
      * A call to the `os.fsencode` function.
      *
@@ -395,7 +406,7 @@ module StdlibPrivate {
      *
      * See https://docs.python.org/3/library/os.html#os.open
      */
-    private class OsOpenCall extends FileSystemAccess::Range, DataFlow::CallCfgNode {
+    class OsOpenCall extends FileSystemAccess::Range, DataFlow::CallCfgNode {
       OsOpenCall() { this = os().getMember("open").getACall() }
 
       override DataFlow::Node getAPathArgument() {
@@ -1492,6 +1503,9 @@ module StdlibPrivate {
     or
     // io.open is a special case, since it is an alias for the builtin `open`
     result = API::moduleImport("io").getMember("open")
+    or
+    // similarly, coecs.open calls the builtin `open`: https://github.com/python/cpython/blob/3.12/Lib/codecs.py#L918
+    result = API::moduleImport("codecs").getMember("open")
   }
 
   /**
@@ -1499,13 +1513,22 @@ module StdlibPrivate {
    * See https://docs.python.org/3/library/functions.html#open
    */
   private class OpenCall extends FileSystemAccess::Range, Stdlib::FileLikeObject::InstanceSource,
-    DataFlow::CallCfgNode
+    ThreatModelSource::Range, DataFlow::CallCfgNode
   {
-    OpenCall() { this = getOpenFunctionRef().getACall() }
+    OpenCall() {
+      this = getOpenFunctionRef().getACall() and
+      // when analyzing stdlib code for os.py we wrongly assume that `os.open` is an
+      // alias of the builtins `open` function
+      not this instanceof OsFileSystemAccessModeling::OsOpenCall
+    }
 
     override DataFlow::Node getAPathArgument() {
       result in [this.getArg(0), this.getArgByName("file")]
     }
+
+    override string getThreatModel() { result = "file" }
+
+    override string getSourceType() { result = "open()" }
   }
 
   /**
@@ -3251,8 +3274,13 @@ module StdlibPrivate {
 
     override predicate propagatesFlow(string input, string output, boolean preservesValue) {
       input in ["Argument[0]", "Argument[pattern:]"] and
-      output = "ReturnValue.Attribute[pattern]" and
-      preservesValue = true
+      (
+        output = "ReturnValue.Attribute[pattern]" and
+        preservesValue = true
+        or
+        output = "ReturnValue" and
+        preservesValue = false
+      )
     }
   }
 
@@ -4207,7 +4235,11 @@ module StdlibPrivate {
   // ---------------------------------------------------------------------------
   // Flow summaries for functions contructing containers
   // ---------------------------------------------------------------------------
-  /** A flow summary for `dict`. */
+  /**
+   * A flow summary for `dict`.
+   *
+   * see https://docs.python.org/3/library/stdtypes.html#dict
+   */
   class DictSummary extends SummarizedCallable {
     DictSummary() { this = "builtins.dict" }
 
@@ -4218,22 +4250,28 @@ module StdlibPrivate {
     }
 
     override predicate propagatesFlow(string input, string output, boolean preservesValue) {
+      // The positional argument contains a mapping.
+      // TODO: these values can be overwritten by keyword arguments
+      //  - dict mapping
       exists(DataFlow::DictionaryElementContent dc, string key | key = dc.getKey() |
         input = "Argument[0].DictionaryElement[" + key + "]" and
         output = "ReturnValue.DictionaryElement[" + key + "]" and
         preservesValue = true
       )
       or
+      //  - list-of-pairs mapping
       input = "Argument[0].ListElement.TupleElement[1]" and
       output = "ReturnValue.DictionaryElementAny" and
       preservesValue = true
       or
+      // The keyword arguments are added to the dictionary.
       exists(DataFlow::DictionaryElementContent dc, string key | key = dc.getKey() |
         input = "Argument[" + key + ":]" and
         output = "ReturnValue.DictionaryElement[" + key + "]" and
         preservesValue = true
       )
       or
+      // Imprecise content in any argument ends up on the container itself.
       input = "Argument[0]" and
       output = "ReturnValue" and
       preservesValue = false
@@ -4993,6 +5031,39 @@ module StdlibPrivate {
 
     override string getKind() { result = Escaping::getHtmlKind() }
   }
+
+  // ---------------------------------------------------------------------------
+  // argparse
+  // ---------------------------------------------------------------------------
+  /**
+   * if result of `parse_args` is tainted (because it uses command-line arguments),
+   *    then the parsed values accesssed on any attribute lookup is also tainted.
+   */
+  private class ArgumentParserAnyAttributeStep extends TaintTracking::AdditionalTaintStep {
+    override predicate step(DataFlow::Node nodeFrom, DataFlow::Node nodeTo) {
+      nodeFrom =
+        API::moduleImport("argparse")
+            .getMember("ArgumentParser")
+            .getReturn()
+            .getMember("parse_args")
+            .getReturn()
+            .getAValueReachableFromSource() and
+      nodeTo.(DataFlow::AttrRead).getObject() = nodeFrom
+    }
+  }
+
+  // ---------------------------------------------------------------------------
+  // sys
+  // ---------------------------------------------------------------------------
+  /**
+   * An access of `sys.stdin`/`sys.stdout`/`sys.stderr`, to get additional FileLike
+   * modeling.
+   */
+  private class SysStandardStreams extends Stdlib::FileLikeObject::InstanceSource, DataFlow::Node {
+    SysStandardStreams() {
+      this = API::moduleImport("sys").getMember(["stdin", "stdout", "stderr"]).asSource()
+    }
+  }
 }
 
 // ---------------------------------------------------------------------------

python/ql/lib/semmle/python/frameworks/data/ModelsAsData.qll

@@ -18,14 +18,19 @@ private import semmle.python.dataflow.new.RemoteFlowSources
 private import semmle.python.dataflow.new.DataFlow
 private import semmle.python.ApiGraphs
 private import semmle.python.dataflow.new.FlowSummary
+private import semmle.python.Concepts
 
 /**
- * A remote flow source originating from a CSV source row.
+ * A threat-model flow source originating from a data extension.
  */
-private class RemoteFlowSourceFromCsv extends RemoteFlowSource::Range {
-  RemoteFlowSourceFromCsv() { this = ModelOutput::getASourceNode("remote").asSource() }
+private class ThreatModelSourceFromDataExtension extends ThreatModelSource::Range {
+  ThreatModelSourceFromDataExtension() { this = ModelOutput::getASourceNode(_).asSource() }
 
-  override string getSourceType() { result = "Remote flow (from model)" }
+  override string getThreatModel() { this = ModelOutput::getASourceNode(result).asSource() }
+
+  override string getSourceType() {
+    result = "Source node (" + this.getThreatModel() + ") [from data-extension]"
+  }
 }
 
 private class SummarizedCallableFromModel extends SummarizedCallable {

python/ql/lib/semmle/python/security/dataflow/CodeInjectionCustomizations.qll

@@ -33,9 +33,14 @@ module CodeInjection {
   abstract class Sanitizer extends DataFlow::Node { }
 
   /**
-   * A source of remote user input, considered as a flow source.
+   * DEPRECATED: Use `ActiveThreatModelSource` from Concepts instead!
    */
-  class RemoteFlowSourceAsSource extends Source, RemoteFlowSource { }
+  deprecated class RemoteFlowSourceAsSource = ActiveThreatModelSourceAsSource;
+
+  /**
+   * An active threat-model source, considered as a flow source.
+   */
+  private class ActiveThreatModelSourceAsSource extends Source, ActiveThreatModelSource { }
 
   /**
    * A code execution, considered as a flow sink.

python/ql/lib/semmle/python/security/dataflow/CommandInjectionCustomizations.qll

@@ -33,9 +33,14 @@ module CommandInjection {
   abstract class Sanitizer extends DataFlow::Node { }
 
   /**
-   * A source of remote user input, considered as a flow source.
+   * DEPRECATED: Use `ActiveThreatModelSource` from Concepts instead!
    */
-  class RemoteFlowSourceAsSource extends Source, RemoteFlowSource { }
+  deprecated class RemoteFlowSourceAsSource = ActiveThreatModelSourceAsSource;
+
+  /**
+   * An active threat-model source, considered as a flow source.
+   */
+  private class ActiveThreatModelSourceAsSource extends Source, ActiveThreatModelSource { }
 
   /**
    * A command execution, considered as a flow sink.

python/ql/lib/semmle/python/security/dataflow/CookieInjectionCustomizations.qll

@@ -31,9 +31,14 @@ module CookieInjection {
   abstract class Sanitizer extends DataFlow::Node { }
 
   /**
-   * A source of remote user input, considered as a flow source.
+   * DEPRECATED: Use `ActiveThreatModelSource` from Concepts instead!
    */
-  class RemoteFlowSourceAsSource extends Source, RemoteFlowSource { }
+  deprecated class RemoteFlowSourceAsSource = ActiveThreatModelSourceAsSource;
+
+  /**
+   * An active threat-model source, considered as a flow source.
+   */
+  private class ActiveThreatModelSourceAsSource extends Source, ActiveThreatModelSource { }
 
   /**
    * A write to a cookie, considered as a sink.

python/ql/lib/semmle/python/security/dataflow/HttpHeaderInjectionCustomizations.qll

@@ -32,9 +32,14 @@ module HttpHeaderInjection {
   abstract class Sanitizer extends DataFlow::Node { }
 
   /**
-   * A source of remote user input, considered as a flow source.
+   * DEPRECATED: Use `ActiveThreatModelSource` from Concepts instead!
    */
-  class RemoteFlowSourceAsSource extends Source, RemoteFlowSource { }
+  deprecated class RemoteFlowSourceAsSource = ActiveThreatModelSourceAsSource;
+
+  /**
+   * An active threat-model source, considered as a flow source.
+   */
+  private class ActiveThreatModelSourceAsSource extends Source, ActiveThreatModelSource { }
 
   /**
    * A HTTP header write, considered as a flow sink.

python/ql/lib/semmle/python/security/dataflow/LdapInjectionCustomizations.qll

@@ -42,9 +42,14 @@ module LdapInjection {
   abstract class FilterSanitizer extends DataFlow::Node { }
 
   /**
-   * A source of remote user input, considered as a flow source.
+   * DEPRECATED: Use `ActiveThreatModelSource` from Concepts instead!
    */
-  class RemoteFlowSourceAsSource extends Source, RemoteFlowSource { }
+  deprecated class RemoteFlowSourceAsSource = ActiveThreatModelSourceAsSource;
+
+  /**
+   * An active threat-model source, considered as a flow source.
+   */
+  private class ActiveThreatModelSourceAsSource extends Source, ActiveThreatModelSource { }
 
   /**
    * A logging operation, considered as a flow sink.

python/ql/lib/semmle/python/security/dataflow/LogInjectionCustomizations.qll

@@ -33,9 +33,14 @@ module LogInjection {
   abstract class Sanitizer extends DataFlow::Node { }
 
   /**
-   * A source of remote user input, considered as a flow source.
+   * DEPRECATED: Use `ActiveThreatModelSource` from Concepts instead!
    */
-  class RemoteFlowSourceAsSource extends Source, RemoteFlowSource { }
+  deprecated class RemoteFlowSourceAsSource = ActiveThreatModelSourceAsSource;
+
+  /**
+   * An active threat-model source, considered as a flow source.
+   */
+  private class ActiveThreatModelSourceAsSource extends Source, ActiveThreatModelSource { }
 
   /**
    * A logging operation, considered as a flow sink.

python/ql/lib/semmle/python/security/dataflow/PamAuthorizationCustomizations.qll

@@ -7,6 +7,7 @@ import python
 import semmle.python.ApiGraphs
 import semmle.python.dataflow.new.TaintTracking
 import semmle.python.dataflow.new.RemoteFlowSources
+import semmle.python.Concepts
 
 /**
  * Provides default sources, sinks and sanitizers for detecting
@@ -39,9 +40,14 @@ module PamAuthorizationCustomizations {
   abstract class Sink extends DataFlow::Node { }
 
   /**
-   * A source of remote user input, considered as a flow source.
+   * DEPRECATED: Use `ActiveThreatModelSource` from Concepts instead!
    */
-  class RemoteFlowSourceAsSource extends Source, RemoteFlowSource { }
+  deprecated class RemoteFlowSourceAsSource = ActiveThreatModelSourceAsSource;
+
+  /**
+   * An active threat-model source, considered as a flow source.
+   */
+  private class ActiveThreatModelSourceAsSource extends Source, ActiveThreatModelSource { }
 
   /**
    * A vulnerable `pam_authenticate` call considered as a flow sink.

python/ql/lib/semmle/python/security/dataflow/PathInjectionCustomizations.qll

@@ -43,9 +43,14 @@ module PathInjection {
   abstract class Sanitizer extends DataFlow::Node { }
 
   /**
-   * A source of remote user input, considered as a flow source.
+   * DEPRECATED: Use `ActiveThreatModelSource` from Concepts instead!
    */
-  class RemoteFlowSourceAsSource extends Source, RemoteFlowSource { }
+  deprecated class RemoteFlowSourceAsSource = ActiveThreatModelSourceAsSource;
+
+  /**
+   * An active threat-model source, considered as a flow source.
+   */
+  private class ActiveThreatModelSourceAsSource extends Source, ActiveThreatModelSource { }
 
   /**
    * A file system access, considered as a flow sink.

python/ql/lib/semmle/python/security/dataflow/PolynomialReDoSCustomizations.qll

@@ -47,9 +47,14 @@ module PolynomialReDoS {
   abstract class Sanitizer extends DataFlow::Node { }
 
   /**
-   * A source of remote user input, considered as a flow source.
+   * DEPRECATED: Use `ActiveThreatModelSource` from Concepts instead!
    */
-  class RemoteFlowSourceAsSource extends Source, RemoteFlowSource { }
+  deprecated class RemoteFlowSourceAsSource = ActiveThreatModelSourceAsSource;
+
+  /**
+   * An active threat-model source, considered as a flow source.
+   */
+  private class ActiveThreatModelSourceAsSource extends Source, ActiveThreatModelSource { }
 
   /**
    * A regex execution, considered as a flow sink.

python/ql/lib/semmle/python/security/dataflow/ReflectedXSSCustomizations.qll

@@ -33,9 +33,14 @@ module ReflectedXss {
   abstract class Sanitizer extends DataFlow::Node { }
 
   /**
-   * A source of remote user input, considered as a flow source.
+   * DEPRECATED: Use `ActiveThreatModelSource` from Concepts instead!
    */
-  class RemoteFlowSourceAsSource extends Source, RemoteFlowSource { }
+  deprecated class RemoteFlowSourceAsSource = ActiveThreatModelSourceAsSource;
+
+  /**
+   * An active threat-model source, considered as a flow source.
+   */
+  private class ActiveThreatModelSourceAsSource extends Source, ActiveThreatModelSource { }
 
   /**
    * A data flow sink for "reflected cross-site scripting" vulnerabilities.

python/ql/lib/semmle/python/security/dataflow/RegexInjectionCustomizations.qll

@@ -40,9 +40,14 @@ module RegexInjection {
   abstract class Sanitizer extends DataFlow::Node { }
 
   /**
-   * A source of remote user input, considered as a flow source.
+   * DEPRECATED: Use `ActiveThreatModelSource` from Concepts instead!
    */
-  class RemoteFlowSourceAsSource extends Source, RemoteFlowSource { }
+  deprecated class RemoteFlowSourceAsSource = ActiveThreatModelSourceAsSource;
+
+  /**
+   * An active threat-model source, considered as a flow source.
+   */
+  private class ActiveThreatModelSourceAsSource extends Source, ActiveThreatModelSource { }
 
   /**
    * A regex escaping, considered as a sanitizer.

python/ql/lib/semmle/python/security/dataflow/ServerSideRequestForgeryCustomizations.qll

@@ -45,9 +45,14 @@ module ServerSideRequestForgery {
   abstract class FullUrlControlSanitizer extends DataFlow::Node { }
 
   /**
-   * A source of remote user input, considered as a flow source.
+   * DEPRECATED: Use `ActiveThreatModelSource` from Concepts instead!
    */
-  class RemoteFlowSourceAsSource extends Source, RemoteFlowSource { }
+  deprecated class RemoteFlowSourceAsSource = ActiveThreatModelSourceAsSource;
+
+  /**
+   * An active threat-model source, considered as a flow source.
+   */
+  private class ActiveThreatModelSourceAsSource extends Source, ActiveThreatModelSource { }
 
   /** The URL of an HTTP request, considered as a sink. */
   class HttpRequestUrlAsSink extends Sink {

python/ql/lib/semmle/python/security/dataflow/SqlInjectionCustomizations.qll

@@ -32,9 +32,14 @@ module SqlInjection {
   abstract class Sanitizer extends DataFlow::Node { }
 
   /**
-   * A source of remote user input, considered as a flow source.
+   * DEPRECATED: Use `ActiveThreatModelSource` from Concepts instead!
    */
-  class RemoteFlowSourceAsSource extends Source, RemoteFlowSource { }
+  deprecated class RemoteFlowSourceAsSource = ActiveThreatModelSourceAsSource;
+
+  /**
+   * An active threat-model source, considered as a flow source.
+   */
+  private class ActiveThreatModelSourceAsSource extends Source, ActiveThreatModelSource { }
 
   /**
    * A SQL statement of a SQL construction, considered as a flow sink.

python/ql/lib/semmle/python/security/dataflow/UnsafeDeserializationCustomizations.qll

@@ -33,9 +33,14 @@ module UnsafeDeserialization {
   abstract class Sanitizer extends DataFlow::Node { }
 
   /**
-   * A source of remote user input, considered as a flow source.
+   * DEPRECATED: Use `ActiveThreatModelSource` from Concepts instead!
    */
-  class RemoteFlowSourceAsSource extends Source, RemoteFlowSource { }
+  deprecated class RemoteFlowSourceAsSource = ActiveThreatModelSourceAsSource;
+
+  /**
+   * An active threat-model source, considered as a flow source.
+   */
+  private class ActiveThreatModelSourceAsSource extends Source, ActiveThreatModelSource { }
 
   /**
    * An insecure decoding, considered as a flow sink.

python/ql/lib/semmle/python/security/dataflow/UnsafeShellCommandConstructionQuery.qll

@@ -45,6 +45,7 @@ module UnsafeShellCommandConstructionConfig implements DataFlow::ConfigSig {
   predicate isSink(DataFlow::Node sink) { sink instanceof Sink }
 
   predicate isBarrier(DataFlow::Node node) {
+    node instanceof Sanitizer or
     node instanceof CommandInjection::Sanitizer // using all sanitizers from `py/command-injection`
   }
 

python/ql/lib/semmle/python/security/dataflow/UrlRedirectCustomizations.qll

@@ -77,9 +77,14 @@ module UrlRedirect {
   }
 
   /**
-   * A source of remote user input, considered as a flow source.
+   * DEPRECATED: Use `ActiveThreatModelSource` from Concepts instead!
    */
-  class RemoteFlowSourceAsSource extends Source, RemoteFlowSource { }
+  deprecated class RemoteFlowSourceAsSource = ActiveThreatModelSourceAsSource;
+
+  /**
+   * An active threat-model source, considered as a flow source.
+   */
+  private class ActiveThreatModelSourceAsSource extends Source, ActiveThreatModelSource { }
 
   /**
    * A HTTP redirect response, considered as a flow sink.

python/ql/lib/semmle/python/security/dataflow/XpathInjectionCustomizations.qll

@@ -30,9 +30,14 @@ module XpathInjection {
   abstract class Sanitizer extends DataFlow::Node { }
 
   /**
-   * A source of remote user input, considered as a flow source.
+   * DEPRECATED: Use `ActiveThreatModelSource` from Concepts instead!
    */
-  class RemoteFlowSourceAsSource extends Source, RemoteFlowSource { }
+  deprecated class RemoteFlowSourceAsSource = ActiveThreatModelSourceAsSource;
+
+  /**
+   * An active threat-model source, considered as a flow source.
+   */
+  private class ActiveThreatModelSourceAsSource extends Source, ActiveThreatModelSource { }
 
   /**
    * A construction of an XPath expression, considered as a sink.