Merge remote-tracking branch 'origin/main' into redsun82/just2

python/ql/src/Statements/ModificationOfLocals.ql

@@ -12,10 +12,10 @@
  */
 
 import python
-private import LegacyPointsTo
+private import semmle.python.ApiGraphs
 
-predicate originIsLocals(ControlFlowNodeWithPointsTo n) {
-  n.pointsTo(_, _, Value::named("locals").getACall())
+predicate originIsLocals(ControlFlowNode n) {
+  API::builtin("locals").getReturn().getAValueReachableFromSource().asCfgNode() = n
 }
 
 predicate modification_of_locals(ControlFlowNode f) {
@@ -37,5 +37,8 @@ where
   // in module level scope `locals() == globals()`
   // see https://docs.python.org/3/library/functions.html#locals
   // FP report in https://github.com/github/codeql/issues/6674
-  not a.getScope() instanceof ModuleScope
+  not a.getScope() instanceof Module and
+  // in class level scope `locals()` reflects the class namespace,
+  // so modifications do take effect.
+  not a.getScope() instanceof Class
 select a, "Modification of the locals() dictionary will have no effect on the local variables."

python/ql/src/Statements/UnreachableCode.ql

@@ -13,7 +13,7 @@
  */
 
 import python
-private import LegacyPointsTo
+private import semmle.python.ApiGraphs
 
 predicate typing_import(ImportingStmt is) {
   exists(Module m |
@@ -34,11 +34,7 @@ predicate unique_yield(Stmt s) {
 /** Holds if `contextlib.suppress` may be used in the same scope as `s` */
 predicate suppression_in_scope(Stmt s) {
   exists(With w |
-    w.getContextExpr()
-        .(Call)
-        .getFunc()
-        .(ExprWithPointsTo)
-        .pointsTo(Value::named("contextlib.suppress")) and
+    w.getContextExpr() = API::moduleImport("contextlib").getMember("suppress").getACall().asExpr() and
     w.getScope() = s.getScope()
   )
 }

python/ql/src/Statements/UnusedExceptionObject.ql

@@ -12,11 +12,49 @@
  */
 
 import python
-private import LegacyPointsTo
+private import semmle.python.dataflow.new.internal.DataFlowDispatch
+private import semmle.python.dataflow.new.internal.Builtins
+private import semmle.python.ApiGraphs
 
-from Call call, ClassValue ex
+/**
+ * Holds if `cls` is a user-defined exception class, i.e. it transitively
+ * extends one of the builtin exception base classes.
+ */
+predicate isUserDefinedExceptionClass(Class cls) {
+  cls.getABase() =
+    API::builtin(["BaseException", "Exception"]).getAValueReachableFromSource().asExpr()
+  or
+  isUserDefinedExceptionClass(getADirectSuperclass(cls))
+}
+
+/**
+ * Gets the name of a builtin exception class.
+ */
+string getBuiltinExceptionName() {
+  result = Builtins::getBuiltinName() and
+  (
+    result.matches("%Error") or
+    result.matches("%Exception") or
+    result.matches("%Warning") or
+    result =
+      ["GeneratorExit", "KeyboardInterrupt", "StopIteration", "StopAsyncIteration", "SystemExit"]
+  )
+}
+
+/**
+ * Holds if `call` is an instantiation of an exception class.
+ */
+predicate isExceptionInstantiation(Call call) {
+  exists(Class cls |
+    classTracker(cls).asExpr() = call.getFunc() and
+    isUserDefinedExceptionClass(cls)
+  )
+  or
+  call.getFunc() = API::builtin(getBuiltinExceptionName()).getAValueReachableFromSource().asExpr()
+}
+
+from Call call
 where
-  call.getFunc().(ExprWithPointsTo).pointsTo(ex) and
-  ex.getASuperType() = ClassValue::exception() and
+  isExceptionInstantiation(call) and
   exists(ExprStmt s | s.getValue() = call)
 select call, "Instantiating an exception, but not raising it, has no effect."

python/ql/src/Statements/UseOfExit.ql

@@ -12,10 +12,12 @@
  */
 
 import python
-private import LegacyPointsTo
+private import semmle.python.ApiGraphs
 
 from CallNode call, string name
-where call.getFunction().(ControlFlowNodeWithPointsTo).pointsTo(Value::siteQuitter(name))
+where
+  name = ["exit", "quit"] and
+  call = API::builtin(name).getACall().asCfgNode()
 select call,
   "The '" + name +
     "' site.Quitter object may not exist if the 'site' module is not loaded or is modified."

python/ql/test/query-tests/Statements/general/test.py

@@ -174,3 +174,9 @@ def assert_ok(seq):
 # False positive. ODASA-8042. Fixed in PR #2401.
 class false_positive:
     e = (x for x in [])
+
+# In class-level scope `locals()` reflects the class namespace,
+# so modifications do take effect.
+class MyClass:
+    locals()['x'] = 43  # OK
+    y = x

python/scripts/create-extractor-pack.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+#
+# Build a local Python extractor pack from source.
+#
+# Usage with the CodeQL CLI (run from the repository root):
+#
+#   codeql database create <db> -l python -s <src> --search-path .
+#   codeql test run --search-path . python/ql/test/<test-dir>
+#
+set -eux
+
+if [[ "$OSTYPE" == "linux-gnu"* ]]; then
+  platform="linux64"
+elif [[ "$OSTYPE" == "darwin"* ]]; then
+  platform="osx64"
+else
+  echo "Unknown OS"
+  exit 1
+fi
+
+cd "$(dirname "$0")/.."
+
+# Build the tsg-python Rust binary
+(cd extractor/tsg-python && cargo build --release)
+tsg_bin="extractor/tsg-python/target/release/tsg-python"
+
+# Generate python3src.zip from the Python extractor source.
+# make_zips.py creates the zip in the source directory and then copies it to the
+# given output directory. We use a temporary directory to avoid a same-file copy
+# error, then move the zip back.
+tmpdir=$(mktemp -d)
+trap 'rm -rf "$tmpdir"' EXIT
+(cd extractor && python3 make_zips.py "$tmpdir")
+cp "$tmpdir/python3src.zip" extractor/python3src.zip
+
+# Assemble the extractor pack
+rm -rf extractor-pack
+mkdir -p extractor-pack/tools/${platform}
+
+# Root-level metadata and schema files
+cp codeql-extractor.yml extractor-pack/
+cp ql/lib/semmlecode.python.dbscheme extractor-pack/
+cp ql/lib/semmlecode.python.dbscheme.stats extractor-pack/
+
+# Python extractor engine files (into tools/)
+cp extractor/python_tracer.py extractor-pack/tools/
+cp extractor/index.py extractor-pack/tools/
+cp extractor/setup.py extractor-pack/tools/
+cp extractor/convert_setup.py extractor-pack/tools/
+cp extractor/get_venv_lib.py extractor-pack/tools/
+cp extractor/imp.py extractor-pack/tools/
+cp extractor/LICENSE-PSF.md extractor-pack/tools/
+cp extractor/python3src.zip extractor-pack/tools/
+cp -r extractor/data extractor-pack/tools/
+
+# Shell tool scripts (autobuild, pre-finalize, lgtm-scripts)
+cp tools/autobuild.sh extractor-pack/tools/
+cp tools/autobuild.cmd extractor-pack/tools/
+cp tools/pre-finalize.sh extractor-pack/tools/
+cp tools/pre-finalize.cmd extractor-pack/tools/
+cp -r tools/lgtm-scripts extractor-pack/tools/
+
+# Downgrades
+cp -r downgrades extractor-pack/
+
+# Platform-specific Rust binary
+cp "${tsg_bin}" extractor-pack/tools/${platform}/tsg-python