Merge pull request #7285 from github/post-release-prep-2.7.3-ddd4ccbb

Post-release preparation 2.7.3

.codeqlmanifest.json

@@ -10,7 +10,14 @@
         "javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml",
         "misc/legacy-support/*/qlpack.yml",
         "misc/suite-helpers/qlpack.yml",
-        "ruby/ql/consistency-queries/qlpack.yml",
-        "ruby/extractor-pack/codeql-extractor.yml"
-   ]
-}
+        "ruby/extractor-pack/codeql-extractor.yml",
+        "ruby/ql/consistency-queries/qlpack.yml"
+    ],
+    "versionPolicies": {
+      "default": {
+        "requireChangeNotes": true,
+        "committedPrereleaseSuffix": "dev",
+        "committedVersion": "nextPatchRelease"
+      }
+    }
+}

cpp/ql/lib/CHANGELOG.md

@@ -0,0 +1,7 @@
+## 0.0.4
+
+### New Features
+
+* The QL library `semmle.code.cpp.commons.Exclusions` now contains a predicate
+  `isFromSystemMacroDefinition` for identifying code that originates from a
+  macro outside the project being analyzed.

cpp/ql/lib/change-notes/released/0.0.4.md

@@ -0,0 +1,7 @@
+## 0.0.4
+
+### New Features
+
+* The QL library `semmle.code.cpp.commons.Exclusions` now contains a predicate
+  `isFromSystemMacroDefinition` for identifying code that originates from a
+  macro outside the project being analyzed.

cpp/ql/lib/codeql-pack.release.yml

@@ -0,0 +1,2 @@
+---
+lastReleaseVersion: 0.0.4

cpp/ql/lib/qlpack.yml

@@ -1,7 +1,8 @@
 name: codeql/cpp-all
-version: 0.0.2
+version: 0.0.5-dev
+groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp
 library: true
 dependencies:
-  codeql/cpp-upgrades: 0.0.2
+  codeql/cpp-upgrades: 0.0.3

cpp/ql/src/CHANGELOG.md

@@ -0,0 +1,5 @@
+## 0.0.4
+
+### New Queries
+
+* A new query `cpp/non-https-url` has been added for C/C++. The query flags uses of `http` URLs that might be better replaced with `https`.

cpp/ql/src/change-notes/released/0.0.4.md

@@ -0,0 +1,5 @@
+## 0.0.4
+
+### New Queries
+
+* A new query `cpp/non-https-url` has been added for C/C++. The query flags uses of `http` URLs that might be better replaced with `https`.

cpp/ql/src/codeql-pack.release.yml

@@ -0,0 +1,2 @@
+---
+lastReleaseVersion: 0.0.4

cpp/ql/src/qlpack.yml

@@ -1,5 +1,6 @@
 name: codeql/cpp-queries
-version: 0.0.2
+version: 0.0.5-dev
+groups: cpp
 dependencies:
     codeql/cpp-all: "*"
     codeql/suite-helpers: "*"

cpp/ql/test/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-tests
-version: 0.0.2
+groups: [cpp, test]
 dependencies:
   codeql/cpp-all: "*"
   codeql/cpp-queries: "*"

cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml

@@ -1,6 +1,6 @@
 # This directory has its own qlpack for reasons detailed in commit 2550788598010fa2117274607c9d58f64f997f34
 name: codeql/cpp-tests-cwe-190-tainted
-version: 0.0.2
+groups: [cpp, test]
 dependencies:
   codeql/cpp-all: "*"
   codeql/cpp-queries: "*"

cpp/upgrades/CHANGELOG.md

@@ -0,0 +1 @@
+## 0.0.4

cpp/upgrades/change-notes/released/0.0.4.md

@@ -0,0 +1 @@
+## 0.0.4

cpp/upgrades/codeql-pack.release.yml

@@ -0,0 +1,2 @@
+---
+lastReleaseVersion: 0.0.4

cpp/upgrades/qlpack.yml

@@ -1,4 +1,5 @@
 name: codeql/cpp-upgrades
+groups: cpp
 upgrades: .
-version: 0.0.2
+version: 0.0.5-dev
 library: true

csharp/ql/lib/CHANGELOG.md

@@ -0,0 +1 @@
+## 0.0.4

csharp/ql/lib/change-notes/released/0.0.4.md

@@ -0,0 +1 @@
+## 0.0.4

csharp/ql/lib/codeql-pack.release.yml

@@ -0,0 +1,2 @@
+---
+lastReleaseVersion: 0.0.4

csharp/ql/lib/qlpack.yml

@@ -1,7 +1,8 @@
 name: codeql/csharp-all
-version: 0.0.2
+version: 0.0.5-dev
+groups: csharp
 dbscheme: semmlecode.csharp.dbscheme
 extractor: csharp
 library: true
 dependencies:
-  codeql/csharp-upgrades: 0.0.2
+  codeql/csharp-upgrades: 0.0.3

csharp/ql/src/CHANGELOG.md

@@ -0,0 +1 @@
+## 0.0.4

csharp/ql/src/change-notes/released/0.0.4.md

@@ -0,0 +1 @@
+## 0.0.4

csharp/ql/src/codeql-pack.release.yml

@@ -0,0 +1,2 @@
+---
+lastReleaseVersion: 0.0.4

csharp/ql/src/qlpack.yml

@@ -1,5 +1,6 @@
 name: codeql/csharp-queries
-version: 0.0.2
+version: 0.0.5-dev
+groups: csharp
 suites: codeql-suites
 extractor: csharp
 defaultSuiteFile: codeql-suites/csharp-code-scanning.qls

csharp/ql/test/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql-csharp-tests
-version: 0.0.2
+groups: [csharp, test]
 dependencies:
   codeql/csharp-all: "*"
   codeql/csharp-queries: "*"

csharp/upgrades/CHANGELOG.md

@@ -0,0 +1 @@
+## 0.0.4

csharp/upgrades/change-notes/released/0.0.4.md

@@ -0,0 +1 @@
+## 0.0.4

csharp/upgrades/codeql-pack.release.yml

@@ -0,0 +1,2 @@
+---
+lastReleaseVersion: 0.0.4

csharp/upgrades/qlpack.yml

@@ -1,4 +1,5 @@
 name: codeql/csharp-upgrades
+groups: csharp
+version: 0.0.5-dev
 upgrades: .
-version: 0.0.2
 library: true

java/ql/lib/CHANGELOG.md

@@ -0,0 +1,7 @@
+## 0.0.4
+
+### Bug Fixes
+
+* `CharacterLiteral`'s `getCodePointValue` predicate now returns the correct value for UTF-16 surrogates.
+* The `RangeAnalysis` module and the `java/constant-comparison` queries no longer raise false alerts regarding comparisons with Unicode surrogate character literals.
+* The predicate `Method.overrides(Method)` was accidentally transitive. This has been fixed. This fix also affects `Method.overridesOrInstantiates(Method)` and `Method.getASourceOverriddenMethod()`.

java/ql/lib/change-notes/released/0.0.4.md

@@ -0,0 +1,7 @@
+## 0.0.4
+
+### Bug Fixes
+
+* `CharacterLiteral`'s `getCodePointValue` predicate now returns the correct value for UTF-16 surrogates.
+* The `RangeAnalysis` module and the `java/constant-comparison` queries no longer raise false alerts regarding comparisons with Unicode surrogate character literals.
+* The predicate `Method.overrides(Method)` was accidentally transitive. This has been fixed. This fix also affects `Method.overridesOrInstantiates(Method)` and `Method.getASourceOverriddenMethod()`.

java/ql/lib/codeql-pack.release.yml

@@ -0,0 +1,2 @@
+---
+lastReleaseVersion: 0.0.4

java/ql/lib/qlpack.yml

@@ -1,7 +1,8 @@
 name: codeql/java-all
-version: 0.0.2
+version: 0.0.5-dev
+groups: java
 dbscheme: config/semmlecode.dbscheme
 extractor: java
 library: true
 dependencies:
-    codeql/java-upgrades: 0.0.2
+    codeql/java-upgrades: 0.0.3

java/ql/src/CHANGELOG.md

@@ -0,0 +1 @@
+## 0.0.4

java/ql/src/change-notes/released/0.0.4.md

@@ -0,0 +1 @@
+## 0.0.4

java/ql/src/codeql-pack.release.yml

@@ -0,0 +1,2 @@
+---
+lastReleaseVersion: 0.0.4

java/ql/src/qlpack.yml

@@ -1,5 +1,6 @@
 name: codeql/java-queries
-version: 0.0.2
+version: 0.0.5-dev
+groups: java
 suites: codeql-suites
 extractor: java
 defaultSuiteFile: codeql-suites/java-code-scanning.qls

java/ql/test/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/java-tests
-version: 0.0.2
+groups: [java, test]
 dependencies:
     codeql/java-all: "*"
     codeql/java-queries: "*"

java/upgrades/CHANGELOG.md

@@ -0,0 +1 @@
+## 0.0.4

java/upgrades/change-notes/released/0.0.4.md

@@ -0,0 +1 @@
+## 0.0.4

java/upgrades/codeql-pack.release.yml

@@ -0,0 +1,2 @@
+---
+lastReleaseVersion: 0.0.4

java/upgrades/qlpack.yml

@@ -1,4 +1,5 @@
 name: codeql/java-upgrades
+groups: java
 upgrades: .
 library: true
-version: 0.0.2
+version: 0.0.5-dev

javascript/ql/experimental/adaptivethreatmodeling/lib/experimental/adaptivethreatmodeling/EndpointFeatures.qll

@@ -13,7 +13,6 @@ import EndpointScoring
  *
  * This is a single string containing a space-separated list of tokens.
  */
-pragma[inline]
 private string getTokenFeature(DataFlow::Node endpoint, string featureName) {
   // Features for endpoints that are contained within a function.
   exists(DatabaseFeatures::Entity entity | entity = getRepresentativeEntityForEndpoint(endpoint) |
@@ -276,7 +275,6 @@ private string getASupportedFeatureName() {
  * This predicate holds if the generic token-based feature named `featureName` has the value
  * `featureValue` for the endpoint `endpoint`.
  */
-pragma[inline]
 predicate tokenFeatures(DataFlow::Node endpoint, string featureName, string featureValue) {
   ModelScoring::endpoints(endpoint) and
   (

javascript/ql/lib/CHANGELOG.md

@@ -0,0 +1 @@
+## 0.0.5

javascript/ql/lib/change-notes/released/0.0.5.md

@@ -0,0 +1 @@
+## 0.0.5

javascript/ql/lib/codeql-pack.release.yml

@@ -0,0 +1,2 @@
+---
+lastReleaseVersion: 0.0.5

javascript/ql/lib/qlpack.yml

@@ -1,7 +1,8 @@
 name: codeql/javascript-all
-version: 0.0.3
+version: 0.0.5
+groups: javascript
 dbscheme: semmlecode.javascript.dbscheme
 extractor: javascript
 library: true
 dependencies:
-    codeql/javascript-upgrades: 0.0.3
+    codeql/javascript-upgrades: 0.0.4

javascript/ql/src/CHANGELOG.md

@@ -0,0 +1,7 @@
+## 0.0.5
+
+### New Queries
+
+* The `js/sensitive-get-query` query has been added. It highlights GET requests that read sensitive information from the query string.
+* The `js/insufficient-key-size` query has been added. It highlights the creation of cryptographic keys with a short key size.
+* The `js/session-fixation` query has been added. It highlights servers that reuse a session after a user has logged in.

javascript/ql/src/change-notes/released/0.0.5.md

@@ -0,0 +1,7 @@
+## 0.0.5
+
+### New Queries
+
+* The `js/sensitive-get-query` query has been added. It highlights GET requests that read sensitive information from the query string.
+* The `js/insufficient-key-size` query has been added. It highlights the creation of cryptographic keys with a short key size.
+* The `js/session-fixation` query has been added. It highlights servers that reuse a session after a user has logged in.

javascript/ql/src/codeql-pack.release.yml

@@ -0,0 +1,2 @@
+---
+lastReleaseVersion: 0.0.5

javascript/ql/src/qlpack.yml

@@ -1,5 +1,6 @@
 name: codeql/javascript-queries
-version: 0.0.3
+version: 0.0.5
+groups: javascript
 suites: codeql-suites
 extractor: javascript
 defaultSuiteFile: codeql-suites/javascript-code-scanning.qls

javascript/ql/test/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/javascript-tests
-version: 0.0.3
+groups: [javascript, test]
 dependencies:
   codeql/javascript-all: "*"
   codeql/javascript-queries: "*"

javascript/upgrades/CHANGELOG.md

@@ -0,0 +1 @@
+## 0.0.5

javascript/upgrades/change-notes/released/0.0.5.md

@@ -0,0 +1 @@
+## 0.0.5

javascript/upgrades/codeql-pack.release.yml

@@ -0,0 +1,2 @@
+---
+lastReleaseVersion: 0.0.5

javascript/upgrades/qlpack.yml

@@ -1,4 +1,5 @@
 name: codeql/javascript-upgrades
+groups: javascript
 upgrades: .
 library: true
-version: 0.0.3
+version: 0.0.5

misc/suite-helpers/qlpack.yml

@@ -1,2 +1,3 @@
 name: codeql/suite-helpers
-version: 0.0.2
+version: 0.0.3
+groups: shared

python/ql/lib/CHANGELOG.md

@@ -0,0 +1,10 @@
+## 0.0.4
+
+### Major Analysis Improvements
+
+* Added modeling of `os.stat`, `os.lstat`, `os.statvfs`, `os.fstat`, and `os.fstatvfs`, which are new sinks for the _Uncontrolled data used in path expression_ (`py/path-injection`) query.
+* Added modeling of the `posixpath`, `ntpath`, and `genericpath` modules for path operations (although these are not supposed to be used), resulting in new sinks for the _Uncontrolled data used in path expression_ (`py/path-injection`) query.
+* Added modeling of `wsgiref.simple_server` applications, leading to new remote flow sources.
+* Added modeling of `aiopg` for sinks executing SQL.
+* Added modeling of HTTP requests and responses when using `flask_admin` (`Flask-Admin` PyPI package), which leads to additional remote flow sources.
+* Added modeling of the PyPI package `toml`, which provides encoding/decoding of TOML documents, leading to new taint-tracking steps.

python/ql/lib/change-notes/released/0.0.4.md

@@ -0,0 +1,10 @@
+## 0.0.4
+
+### Major Analysis Improvements
+
+* Added modeling of `os.stat`, `os.lstat`, `os.statvfs`, `os.fstat`, and `os.fstatvfs`, which are new sinks for the _Uncontrolled data used in path expression_ (`py/path-injection`) query.
+* Added modeling of the `posixpath`, `ntpath`, and `genericpath` modules for path operations (although these are not supposed to be used), resulting in new sinks for the _Uncontrolled data used in path expression_ (`py/path-injection`) query.
+* Added modeling of `wsgiref.simple_server` applications, leading to new remote flow sources.
+* Added modeling of `aiopg` for sinks executing SQL.
+* Added modeling of HTTP requests and responses when using `flask_admin` (`Flask-Admin` PyPI package), which leads to additional remote flow sources.
+* Added modeling of the PyPI package `toml`, which provides encoding/decoding of TOML documents, leading to new taint-tracking steps.

python/ql/lib/codeql-pack.release.yml

@@ -0,0 +1,2 @@
+---
+lastReleaseVersion: 0.0.4

python/ql/lib/qlpack.yml

@@ -1,7 +1,8 @@
 name: codeql/python-all
-version: 0.0.2
+version: 0.0.5-dev
+groups: python
 dbscheme: semmlecode.python.dbscheme
 extractor: python
 library: true
 dependencies:
-    codeql/python-upgrades: 0.0.2
+    codeql/python-upgrades: 0.0.3

python/ql/lib/semmle/python/ApiGraphs.qll

@@ -358,134 +358,26 @@ module API {
       )
     }
 
-    /** Gets the name of a known built-in. */
-    private string getBuiltInName() {
-      // These lists were created by inspecting the `builtins` and `__builtin__` modules in
-      // Python 3 and 2 respectively, using the `dir` built-in.
-      // Built-in functions and exceptions shared between Python 2 and 3
-      result in [
-          "abs", "all", "any", "bin", "bool", "bytearray", "callable", "chr", "classmethod",
-          "compile", "complex", "delattr", "dict", "dir", "divmod", "enumerate", "eval", "filter",
-          "float", "format", "frozenset", "getattr", "globals", "hasattr", "hash", "help", "hex",
-          "id", "input", "int", "isinstance", "issubclass", "iter", "len", "list", "locals", "map",
-          "max", "memoryview", "min", "next", "object", "oct", "open", "ord", "pow", "print",
-          "property", "range", "repr", "reversed", "round", "set", "setattr", "slice", "sorted",
-          "staticmethod", "str", "sum", "super", "tuple", "type", "vars", "zip", "__import__",
-          // Exceptions
-          "ArithmeticError", "AssertionError", "AttributeError", "BaseException", "BufferError",
-          "BytesWarning", "DeprecationWarning", "EOFError", "EnvironmentError", "Exception",
-          "FloatingPointError", "FutureWarning", "GeneratorExit", "IOError", "ImportError",
-          "ImportWarning", "IndentationError", "IndexError", "KeyError", "KeyboardInterrupt",
-          "LookupError", "MemoryError", "NameError", "NotImplemented", "NotImplementedError",
-          "OSError", "OverflowError", "PendingDeprecationWarning", "ReferenceError", "RuntimeError",
-          "RuntimeWarning", "StandardError", "StopIteration", "SyntaxError", "SyntaxWarning",
-          "SystemError", "SystemExit", "TabError", "TypeError", "UnboundLocalError",
-          "UnicodeDecodeError", "UnicodeEncodeError", "UnicodeError", "UnicodeTranslateError",
-          "UnicodeWarning", "UserWarning", "ValueError", "Warning", "ZeroDivisionError",
-          // Added for compatibility
-          "exec"
-        ]
-      or
-      // Built-in constants shared between Python 2 and 3
-      result in ["False", "True", "None", "NotImplemented", "Ellipsis", "__debug__"]
-      or
-      // Python 3 only
-      result in [
-          "ascii", "breakpoint", "bytes", "exec", "aiter", "anext",
-          // Exceptions
-          "BlockingIOError", "BrokenPipeError", "ChildProcessError", "ConnectionAbortedError",
-          "ConnectionError", "ConnectionRefusedError", "ConnectionResetError", "FileExistsError",
-          "FileNotFoundError", "InterruptedError", "IsADirectoryError", "ModuleNotFoundError",
-          "NotADirectoryError", "PermissionError", "ProcessLookupError", "RecursionError",
-          "ResourceWarning", "StopAsyncIteration", "TimeoutError"
-        ]
-      or
-      // Python 2 only
-      result in [
-          "basestring", "cmp", "execfile", "file", "long", "raw_input", "reduce", "reload",
-          "unichr", "unicode", "xrange"
-        ]
-    }
-
-    /**
-     * Gets a data flow node that is likely to refer to a built-in with the name `name`.
-     *
-     * Currently this is an over-approximation, and may not account for things like overwriting a
-     * built-in with a different value.
-     */
-    private DataFlow::Node likely_builtin(string name) {
-      exists(Module m |
-        result.asCfgNode() =
-          any(NameNode n |
-            possible_builtin_accessed_in_module(n, name, m) and
-            not possible_builtin_defined_in_module(name, m)
-          )
-      )
-    }
-
-    /**
-     * Holds if a global variable called `name` (which is also the name of a built-in) is assigned
-     * a value in the module `m`.
-     */
-    private predicate possible_builtin_defined_in_module(string name, Module m) {
-      global_name_defined_in_module(name, m) and
-      name = getBuiltInName()
-    }
-
-    /**
-     * Holds if `n` is an access of a global variable called `name` (which is also the name of a
-     * built-in) inside the module `m`.
-     */
-    private predicate possible_builtin_accessed_in_module(NameNode n, string name, Module m) {
-      n.isGlobal() and
-      n.isLoad() and
-      name = n.getId() and
-      name = getBuiltInName() and
-      m = n.getEnclosingModule()
-    }
-
-    /**
-     * Holds if `n` is an access of a variable called `name` (which is _not_ the name of a
-     * built-in, and which is _not_ a global defined in the enclosing module) inside the scope `s`.
-     */
-    private predicate name_possibly_defined_in_import_star(NameNode n, string name, Scope s) {
-      n.isLoad() and
-      name = n.getId() and
-      // Not already defined in an enclosing scope.
-      not exists(LocalVariable v |
-        v.getId() = name and v.getScope() = n.getScope().getEnclosingScope*()
-      ) and
-      not name = getBuiltInName() and
-      s = n.getScope().getEnclosingScope*() and
-      exists(potential_import_star_base(s)) and
-      not global_name_defined_in_module(name, n.getEnclosingModule())
-    }
-
-    /** Holds if a global variable called `name` is assigned a value in the module `m`. */
-    private predicate global_name_defined_in_module(string name, Module m) {
-      exists(NameNode n |
-        not exists(LocalVariable v | n.defines(v)) and
-        n.isStore() and
-        name = n.getId() and
-        m = n.getEnclosingModule()
-      )
-    }
+    private import semmle.python.dataflow.new.internal.Builtins
+    private import semmle.python.dataflow.new.internal.ImportStar
 
     /**
      * Gets the API graph node for all modules imported with `from ... import *` inside the scope `s`.
      *
      * For example, given
      *
-     * `from foo.bar import *`
+     * ```python
+     * from foo.bar import *
+     * ```
      *
      * this would be the API graph node with the path
      *
      * `moduleImport("foo").getMember("bar")`
      */
     private TApiNode potential_import_star_base(Scope s) {
-      exists(DataFlow::Node ref |
-        ref.asCfgNode() = any(ImportStarNode n | n.getScope() = s).getModule() and
-        use(result, ref)
+      exists(DataFlow::Node n |
+        n.asCfgNode() = ImportStar::potentialImportStarBase(s) and
+        use(result, n)
       )
     }
 
@@ -529,14 +421,14 @@ module API {
       or
       // Built-ins, treated as members of the module `builtins`
       base = MkModuleImport("builtins") and
-      lbl = Label::member(any(string name | ref = likely_builtin(name)))
+      lbl = Label::member(any(string name | ref = Builtins::likelyBuiltin(name)))
       or
       // Unknown variables that may belong to a module imported with `import *`
       exists(Scope s |
         base = potential_import_star_base(s) and
         lbl =
           Label::member(any(string name |
-              name_possibly_defined_in_import_star(ref.asCfgNode(), name, s)
+              ImportStar::namePossiblyDefinedInImportStar(ref.asCfgNode(), name, s)
             ))
       )
     }

python/ql/lib/semmle/python/dataflow/new/internal/Builtins.qll

@@ -0,0 +1,93 @@
+/** Provides predicates for reasoning about built-ins in Python. */
+
+private import python
+private import semmle.python.dataflow.new.DataFlow
+private import semmle.python.dataflow.new.internal.ImportStar
+
+module Builtins {
+  /** Gets the name of a known built-in. */
+  string getBuiltinName() {
+    // These lists were created by inspecting the `builtins` and `__builtin__` modules in
+    // Python 3 and 2 respectively, using the `dir` built-in.
+    // Built-in functions and exceptions shared between Python 2 and 3
+    result in [
+        "abs", "all", "any", "bin", "bool", "bytearray", "callable", "chr", "classmethod",
+        "compile", "complex", "delattr", "dict", "dir", "divmod", "enumerate", "eval", "filter",
+        "float", "format", "frozenset", "getattr", "globals", "hasattr", "hash", "help", "hex",
+        "id", "input", "int", "isinstance", "issubclass", "iter", "len", "list", "locals", "map",
+        "max", "memoryview", "min", "next", "object", "oct", "open", "ord", "pow", "print",
+        "property", "range", "repr", "reversed", "round", "set", "setattr", "slice", "sorted",
+        "staticmethod", "str", "sum", "super", "tuple", "type", "vars", "zip", "__import__",
+        // Exceptions
+        "ArithmeticError", "AssertionError", "AttributeError", "BaseException", "BufferError",
+        "BytesWarning", "DeprecationWarning", "EOFError", "EnvironmentError", "Exception",
+        "FloatingPointError", "FutureWarning", "GeneratorExit", "IOError", "ImportError",
+        "ImportWarning", "IndentationError", "IndexError", "KeyError", "KeyboardInterrupt",
+        "LookupError", "MemoryError", "NameError", "NotImplemented", "NotImplementedError",
+        "OSError", "OverflowError", "PendingDeprecationWarning", "ReferenceError", "RuntimeError",
+        "RuntimeWarning", "StandardError", "StopIteration", "SyntaxError", "SyntaxWarning",
+        "SystemError", "SystemExit", "TabError", "TypeError", "UnboundLocalError",
+        "UnicodeDecodeError", "UnicodeEncodeError", "UnicodeError", "UnicodeTranslateError",
+        "UnicodeWarning", "UserWarning", "ValueError", "Warning", "ZeroDivisionError",
+        // Added for compatibility
+        "exec"
+      ]
+    or
+    // Built-in constants shared between Python 2 and 3
+    result in ["False", "True", "None", "NotImplemented", "Ellipsis", "__debug__"]
+    or
+    // Python 3 only
+    result in [
+        "ascii", "breakpoint", "bytes", "exec",
+        // Exceptions
+        "BlockingIOError", "BrokenPipeError", "ChildProcessError", "ConnectionAbortedError",
+        "ConnectionError", "ConnectionRefusedError", "ConnectionResetError", "FileExistsError",
+        "FileNotFoundError", "InterruptedError", "IsADirectoryError", "ModuleNotFoundError",
+        "NotADirectoryError", "PermissionError", "ProcessLookupError", "RecursionError",
+        "ResourceWarning", "StopAsyncIteration", "TimeoutError"
+      ]
+    or
+    // Python 2 only
+    result in [
+        "basestring", "cmp", "execfile", "file", "long", "raw_input", "reduce", "reload", "unichr",
+        "unicode", "xrange"
+      ]
+  }
+
+  /**
+   * Gets a data flow node that is likely to refer to a built-in with the name `name`.
+   *
+   * Currently this is an over-approximation, and may not account for things like overwriting a
+   * built-in with a different value.
+   */
+  DataFlow::Node likelyBuiltin(string name) {
+    exists(Module m |
+      result.asCfgNode() =
+        any(NameNode n |
+          possible_builtin_accessed_in_module(n, name, m) and
+          not possible_builtin_defined_in_module(name, m)
+        )
+    )
+  }
+
+  /**
+   * Holds if a global variable called `name` (which is also the name of a built-in) is assigned
+   * a value in the module `m`.
+   */
+  private predicate possible_builtin_defined_in_module(string name, Module m) {
+    ImportStar::globalNameDefinedInModule(name, m) and
+    name = getBuiltinName()
+  }
+
+  /**
+   * Holds if `n` is an access of a global variable called `name` (which is also the name of a
+   * built-in) inside the module `m`.
+   */
+  private predicate possible_builtin_accessed_in_module(NameNode n, string name, Module m) {
+    n.isGlobal() and
+    n.isLoad() and
+    name = n.getId() and
+    name = getBuiltinName() and
+    m = n.getEnclosingModule()
+  }
+}

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPrivate.qll

@@ -2,6 +2,7 @@ private import python
 private import DataFlowPublic
 import semmle.python.SpecialMethods
 private import semmle.python.essa.SsaCompute
+private import semmle.python.dataflow.new.internal.ImportStar
 
 /** Gets the callable in which this node occurs. */
 DataFlowCallable nodeGetEnclosingCallable(Node n) { result = n.getEnclosingCallable() }
@@ -948,7 +949,7 @@ predicate jumpStep(Node nodeFrom, Node nodeTo) {
 private predicate module_export(Module m, string name, CfgNode defn) {
   exists(EssaVariable v |
     v.getName() = name and
-    v.getAUse() = m.getANormalExit()
+    v.getAUse() = ImportStar::getStarImported*(m).getANormalExit()
   |
     defn.getNode() = v.getDefinition().(AssignmentDefinition).getValue()
     or

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPublic.qll

@@ -8,6 +8,7 @@ import semmle.python.dataflow.new.TypeTracker
 import Attributes
 import LocalSources
 private import semmle.python.essa.SsaCompute
+private import semmle.python.dataflow.new.internal.ImportStar
 
 /**
  * IPA type for data flow nodes.
@@ -30,7 +31,15 @@ newtype TNode =
   /** A synthetic node representing the value of an object after a state change. */
   TSyntheticPostUpdateNode(NeedsSyntheticPostUpdateNode pre) or
   /** A node representing a global (module-level) variable in a specific module. */
-  TModuleVariableNode(Module m, GlobalVariable v) { v.getScope() = m and v.escapes() } or
+  TModuleVariableNode(Module m, GlobalVariable v) {
+    v.getScope() = m and
+    (
+      v.escapes()
+      or
+      isAccessedThroughImportStar(m) and
+      ImportStar::globalNameDefinedInModule(v.getId(), m)
+    )
+  } or
   /**
    * A node representing the overflow positional arguments to a call.
    * That is, `call` contains more positional arguments than there are
@@ -346,6 +355,8 @@ class ModuleVariableNode extends Node, TModuleVariableNode {
     result.asCfgNode() = var.getALoad().getAFlowNode() and
     // Ignore reads that happen when the module is imported. These are only executed once.
     not result.getScope() = mod
+    or
+    this = import_star_read(result)
   }
 
   /** Gets an `EssaNode` that corresponds to an assignment of this global variable. */
@@ -358,6 +369,13 @@ class ModuleVariableNode extends Node, TModuleVariableNode {
   override Location getLocation() { result = mod.getLocation() }
 }
 
+private predicate isAccessedThroughImportStar(Module m) { m = ImportStar::getStarImported(_) }
+
+private ModuleVariableNode import_star_read(Node n) {
+  ImportStar::importStarResolvesTo(n.asCfgNode(), result.getModule()) and
+  n.asCfgNode().(NameNode).getId() = result.getVariable().getId()
+}
+
 /**
  * The node holding the extra positional arguments to a call. This node is passed as a tuple
  * to the starred parameter of the callable.

python/ql/lib/semmle/python/dataflow/new/internal/ImportStar.qll

@@ -0,0 +1,95 @@
+/** Provides predicates for reasoning about uses of `import *` in Python. */
+
+private import python
+private import semmle.python.dataflow.new.internal.Builtins
+
+cached
+module ImportStar {
+  /**
+   * Holds if `n` is an access of a variable called `name` (which is _not_ the name of a
+   * built-in, and which is _not_ a global defined in the enclosing module) inside the scope `s`.
+   */
+  cached
+  predicate namePossiblyDefinedInImportStar(NameNode n, string name, Scope s) {
+    n.isLoad() and
+    name = n.getId() and
+    s = n.getScope().getEnclosingScope*() and
+    exists(potentialImportStarBase(s)) and
+    // Not already defined in an enclosing scope.
+    not isDefinedLocally(n.getNode())
+  }
+
+  /** Holds if `n` refers to a variable that is defined in the module in which it occurs. */
+  cached
+  private predicate isDefinedLocally(Name n) {
+    // Defined in an enclosing scope
+    enclosing_scope_defines_name(n.getScope(), n.getId())
+    or
+    // Defined as a built-in
+    n.getId() = Builtins::getBuiltinName()
+    or
+    // Defined as a global in this module
+    globalNameDefinedInModule(n.getId(), n.getEnclosingModule())
+    or
+    // A non-built-in that still has file-specific meaning
+    n.getId() in ["__name__", "__package__"]
+  }
+
+  pragma[nomagic]
+  private predicate enclosing_scope_defines_name(Scope s, string name) {
+    exists(LocalVariable v |
+      v.getId() = name and v.getScope() = s and not name = Builtins::getBuiltinName()
+    )
+    or
+    enclosing_scope_defines_name(s.getEnclosingScope(), name)
+  }
+
+  /** Holds if a global variable called `name` is assigned a value in the module `m`. */
+  cached
+  predicate globalNameDefinedInModule(string name, Module m) {
+    exists(NameNode n |
+      not exists(LocalVariable v | n.defines(v)) and
+      n.isStore() and
+      name = n.getId() and
+      m = n.getEnclosingModule()
+    )
+  }
+
+  /**
+   * Holds if `n` may refer to a global variable of the same name in the module `m`, accessible
+   *  from the scope of `n` by a chain of `import *` imports.
+   */
+  cached
+  predicate importStarResolvesTo(NameNode n, Module m) {
+    m = getStarImported+(n.getEnclosingModule()) and
+    globalNameDefinedInModule(n.getId(), m) and
+    not isDefinedLocally(n.getNode())
+  }
+
+  /**
+   * Gets a module that is imported from `m` via `import *`.
+   */
+  cached
+  Module getStarImported(Module m) {
+    exists(ImportStar i |
+      i.getScope() = m and result = i.getModule().pointsTo().(ModuleValue).getScope()
+    )
+  }
+
+  /**
+   * Gets the data-flow node for a module imported with `from ... import *` inside the scope `s`.
+   *
+   * For example, given
+   *
+   * ```python
+   * from foo.bar import *
+   * from quux import *
+   * ```
+   *
+   * this would return the data-flow nodes corresponding to `foo.bar` and `quux`.
+   */
+  cached
+  ControlFlowNode potentialImportStarBase(Scope s) {
+    result = any(ImportStarNode n | n.getScope() = s).getModule()
+  }
+}

python/ql/src/CHANGELOG.md

@@ -0,0 +1,5 @@
+## 0.0.4
+
+### Query Metadata Changes
+
+* Fixed the query ids of two queries that are meant for manual exploration: `python/count-untrusted-data-external-api` and `python/untrusted-data-to-external-api` have been changed to `py/count-untrusted-data-external-api` and `py/untrusted-data-to-external-api`.

python/ql/src/change-notes/released/0.0.4.md

@@ -0,0 +1,5 @@
+## 0.0.4
+
+### Query Metadata Changes
+
+* Fixed the query ids of two queries that are meant for manual exploration: `python/count-untrusted-data-external-api` and `python/untrusted-data-to-external-api` have been changed to `py/count-untrusted-data-external-api` and `py/untrusted-data-to-external-api`.

python/ql/src/codeql-pack.release.yml

@@ -0,0 +1,2 @@
+---
+lastReleaseVersion: 0.0.4

python/ql/src/qlpack.yml

@@ -1,5 +1,6 @@
 name: codeql/python-queries
-version: 0.0.2
+version: 0.0.5-dev
+groups: python
 dependencies:
     codeql/python-all: "*"
     codeql/suite-helpers: "*"

python/ql/test/experimental/dataflow/global-flow/known.py

@@ -1 +1 @@
-known_attr = [1000]
+known_attr = [1000] #$ writes=known_attr

python/ql/test/experimental/dataflow/import-star/deux.py

@@ -0,0 +1,2 @@
+from trois import *
+print(foo)

python/ql/test/experimental/dataflow/import-star/global.expected

@@ -0,0 +1,15 @@
+| test3.py:1:17:1:19 | ControlFlowNode for ImportMember | test3.py:2:7:2:9 | ControlFlowNode for foo |
+| three.py:1:1:1:3 | ControlFlowNode for foo | test1.py:2:7:2:9 | ControlFlowNode for foo |
+| three.py:1:1:1:3 | ControlFlowNode for foo | test3.py:1:17:1:19 | ControlFlowNode for ImportMember |
+| three.py:1:1:1:3 | ControlFlowNode for foo | test3.py:2:7:2:9 | ControlFlowNode for foo |
+| three.py:1:1:1:3 | ControlFlowNode for foo | two.py:2:7:2:9 | ControlFlowNode for foo |
+| three.py:1:7:1:7 | ControlFlowNode for IntegerLiteral | test1.py:2:7:2:9 | ControlFlowNode for foo |
+| three.py:1:7:1:7 | ControlFlowNode for IntegerLiteral | test3.py:1:17:1:19 | ControlFlowNode for ImportMember |
+| three.py:1:7:1:7 | ControlFlowNode for IntegerLiteral | test3.py:2:7:2:9 | ControlFlowNode for foo |
+| three.py:1:7:1:7 | ControlFlowNode for IntegerLiteral | two.py:2:7:2:9 | ControlFlowNode for foo |
+| trois.py:1:1:1:3 | ControlFlowNode for foo | deux.py:2:7:2:9 | ControlFlowNode for foo |
+| trois.py:1:1:1:3 | ControlFlowNode for foo | test2.py:2:7:2:9 | ControlFlowNode for foo |
+| trois.py:1:7:1:7 | ControlFlowNode for IntegerLiteral | deux.py:2:7:2:9 | ControlFlowNode for foo |
+| trois.py:1:7:1:7 | ControlFlowNode for IntegerLiteral | test2.py:2:7:2:9 | ControlFlowNode for foo |
+| two.py:2:7:2:9 | ControlFlowNode for foo | test3.py:1:17:1:19 | ControlFlowNode for ImportMember |
+| two.py:2:7:2:9 | ControlFlowNode for foo | test3.py:2:7:2:9 | ControlFlowNode for foo |

python/ql/test/experimental/dataflow/import-star/global.ql

@@ -0,0 +1,19 @@
+import semmle.python.dataflow.new.DataFlow
+
+/**
+ * A configuration to find all flows.
+ * To be used on tiny programs.
+ */
+class AllFlowsConfig extends DataFlow::Configuration {
+  AllFlowsConfig() { this = "AllFlowsConfig" }
+
+  override predicate isSource(DataFlow::Node node) { any() }
+
+  override predicate isSink(DataFlow::Node node) { any() }
+}
+
+from DataFlow::CfgNode source, DataFlow::CfgNode sink
+where
+  source != sink and
+  exists(AllFlowsConfig cfg | cfg.hasFlow(source, sink))
+select source, sink

python/ql/test/experimental/dataflow/import-star/one.py

@@ -0,0 +1 @@
+from two import *

python/ql/test/experimental/dataflow/import-star/test1.py

@@ -0,0 +1,2 @@
+from one import *
+print(foo)

python/ql/test/experimental/dataflow/import-star/test2.py

@@ -0,0 +1,2 @@
+from un import *
+print(foo)

python/ql/test/experimental/dataflow/import-star/test3.py

@@ -0,0 +1,2 @@
+from one import foo
+print(foo)

python/ql/test/experimental/dataflow/import-star/three.py

@@ -0,0 +1 @@
+foo = 5

python/ql/test/experimental/dataflow/import-star/trois.py

@@ -0,0 +1 @@
+foo = 6

python/ql/test/experimental/dataflow/import-star/two.py

@@ -0,0 +1,2 @@
+from three import *
+print(foo)

python/ql/test/experimental/dataflow/import-star/un.py

@@ -0,0 +1 @@
+from deux import *

python/ql/test/library-tests/PointsTo/new/ImpliesDataflow.expected

@@ -15,4 +15,3 @@
 | code/r_regressions.py:46:1:46:14 | ControlFlowNode for FunctionExpr | code/r_regressions.py:52:9:52:12 | ControlFlowNode for fail |
 | code/t_type.py:3:1:3:16 | ControlFlowNode for ClassExpr | code/t_type.py:6:1:6:9 | ControlFlowNode for type() |
 | code/t_type.py:3:1:3:16 | ControlFlowNode for ClassExpr | code/t_type.py:13:5:13:13 | ControlFlowNode for type() |
-| code/test_package/module2.py:5:5:5:6 | ControlFlowNode for Dict | code/j_convoluted_imports.py:25:1:25:1 | ControlFlowNode for r |

python/ql/test/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/python-tests
-version: 0.0.2
+groups: [python, test]
 dependencies:
     codeql/python-all: "*"
     codeql/python-queries: "*"

python/upgrades/CHANGELOG.md

@@ -0,0 +1 @@
+## 0.0.4

python/upgrades/change-notes/released/0.0.4.md

@@ -0,0 +1 @@
+## 0.0.4

python/upgrades/codeql-pack.release.yml

@@ -0,0 +1,2 @@
+---
+lastReleaseVersion: 0.0.4

python/upgrades/qlpack.yml

@@ -1,4 +1,5 @@
 name: codeql/python-upgrades
+groups: python
 upgrades: .
 library: true
-version: 0.0.2
+version: 0.0.5-dev

ruby/ql/lib/CHANGELOG.md

@@ -0,0 +1 @@
+## 0.0.4

ruby/ql/lib/change-notes/released/0.0.4.md

@@ -0,0 +1 @@
+## 0.0.4

ruby/ql/lib/codeql-pack.release.yml

@@ -0,0 +1,2 @@
+---
+lastReleaseVersion: 0.0.4

ruby/ql/lib/qlpack.yml

@@ -1,5 +1,6 @@
 name: codeql/ruby-all
-version: 0.0.2
+version: 0.0.5-dev
+groups: ruby
 extractor: ruby
 dbscheme: ruby.dbscheme
 upgrades: upgrades

ruby/ql/src/CHANGELOG.md

@@ -0,0 +1,10 @@
+## 0.0.4
+
+### New Queries
+
+* A new query (`rb/request-forgery`) has been added. The query finds HTTP requests made with user-controlled URLs.
+* A new query (`rb/csrf-protection-disabled`) has been added. The query finds cases where cross-site forgery protection is explictly disabled.
+
+### Query Metadata Changes
+
+* The precision of "Hard-coded credentials" (`rb/hardcoded-credentials`) has been decreased from "high" to "medium". This query will no longer be run and displayed by default on Code Scanning and LGTM.

ruby/ql/src/change-notes/released/0.0.4.md

@@ -0,0 +1,10 @@
+## 0.0.4
+
+### New Queries
+
+* A new query (`rb/request-forgery`) has been added. The query finds HTTP requests made with user-controlled URLs.
+* A new query (`rb/csrf-protection-disabled`) has been added. The query finds cases where cross-site forgery protection is explictly disabled.
+
+### Query Metadata Changes
+
+* The precision of "Hard-coded credentials" (`rb/hardcoded-credentials`) has been decreased from "high" to "medium". This query will no longer be run and displayed by default on Code Scanning and LGTM.

ruby/ql/src/codeql-pack.release.yml

@@ -0,0 +1,2 @@
+---
+lastReleaseVersion: 0.0.4

ruby/ql/src/qlpack.yml

@@ -1,5 +1,6 @@
 name: codeql/ruby-queries
-version: 0.0.2
+version: 0.0.5-dev
+groups: ruby
 suites: codeql-suites
 defaultSuiteFile: codeql-suites/ruby-code-scanning.qls
 dependencies:

ruby/ql/test/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/ruby-tests
-version: 0.0.2
+groups: [ruby, test]
 dependencies:
   codeql/ruby-queries: ^0.0.2
   codeql/ruby-examples: ^0.0.2