Merge rc/1.21 into master

python/ql/src/semmle/python/objects/Callables.qll

@@ -161,6 +161,8 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti
 
     override predicate contextSensitiveCallee() { any() }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
 }
 
 
@@ -284,6 +286,8 @@ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunc
 
     override predicate contextSensitiveCallee() { none() }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
 }
 
 /** Class representing methods of built-in classes (otherwise known as method-descriptors) such as `list.append`.
@@ -376,6 +380,8 @@ class BuiltinMethodObjectInternal extends CallableObjectInternal, TBuiltinMethod
 
     override predicate contextSensitiveCallee() { none() }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
 }
 
 /** Class representing bound-methods.
@@ -461,6 +467,8 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod {
         function = this.getFunction() and offset = 1
     }
 
+    override predicate useOriginAsLegacyObject() { any() }
+
     override predicate contextSensitiveCallee() {
         this.getFunction().contextSensitiveCallee()
     }

python/ql/src/semmle/python/objects/Classes.qll

@@ -92,6 +92,8 @@ abstract class ClassObjectInternal extends ObjectInternal {
 
     override predicate contextSensitiveCallee() { none() }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
     /* Classes aren't usually iterable, but can e.g. Enums */
     override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
 

python/ql/src/semmle/python/objects/Constants.qll

@@ -71,6 +71,8 @@ abstract class ConstantObjectInternal extends ObjectInternal {
 
     override predicate contextSensitiveCallee() { none() }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
 }
 
 private abstract class BooleanObjectInternal extends ConstantObjectInternal {

python/ql/src/semmle/python/objects/Descriptors.qll

@@ -93,6 +93,8 @@ class PropertyInternal extends ObjectInternal, TProperty {
 
     override predicate contextSensitiveCallee() { none() }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
     /* Properties aren't iterable */
     override ObjectInternal getIterNext() { none() }
 
@@ -183,6 +185,8 @@ class ClassMethodObjectInternal extends ObjectInternal, TClassMethod {
 
     override predicate contextSensitiveCallee() { none() }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
     /* Classmethods aren't iterable */
     override ObjectInternal getIterNext() { none() }
 
@@ -259,6 +263,8 @@ class StaticMethodObjectInternal extends ObjectInternal, TStaticMethod {
 
     override predicate contextSensitiveCallee() { none() }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
     /* Staticmethods aren't iterable */
     override ObjectInternal getIterNext() { none() }
 

python/ql/src/semmle/python/objects/Instances.qll

@@ -164,6 +164,8 @@ class SpecificInstanceInternal extends TSpecificInstance, InstanceObject {
         )
     }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
 }
 
 /** A class representing context-free instances represented by `self` in the source code
@@ -266,6 +268,8 @@ class SelfInstanceInternal extends TSelfInstance, InstanceObject {
         this.getClass().attribute("__init__", init, _)
     }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
 }
 
 /** A class representing a value that has a known class, but no other information */
@@ -372,6 +376,8 @@ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal {
 
     override predicate contextSensitiveCallee() { none() }
 
+    override predicate useOriginAsLegacyObject() { any() }
+
     override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
 
 }
@@ -482,6 +488,8 @@ class SuperInstance extends TSuperInstance, ObjectInternal {
 
     override predicate contextSensitiveCallee() { none() }
 
+    override predicate useOriginAsLegacyObject() { any() }
+
     override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
 
 }

python/ql/src/semmle/python/objects/Modules.qll

@@ -54,6 +54,8 @@ abstract class ModuleObjectInternal extends ObjectInternal {
 
     override predicate contextSensitiveCallee() { none() }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
     /* Modules aren't iterable */
     override ObjectInternal getIterNext() { none() }
 
@@ -313,10 +315,6 @@ class AbsentModuleObjectInternal extends ModuleObjectInternal, TAbsentModule {
         none()
     }
 
-    override predicate isMissing() {
-        any()
-    }
-
 }
 
 /** A class representing an attribute of a missing module. */
@@ -402,15 +400,13 @@ class AbsentModuleAttributeObjectInternal extends ObjectInternal, TAbsentModuleA
 
     override predicate subscriptUnknown() { any() }
 
-    override predicate isMissing() {
-        any()
-    }
-
     /* We know what this is called, but not its innate name */
     override string getName() { none() }
 
     override predicate contextSensitiveCallee() { none() }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
     /* Modules aren't iterable */
     override ObjectInternal getIterNext() { none() }
 

python/ql/src/semmle/python/objects/ObjectAPI.qll

@@ -78,14 +78,6 @@ class Value extends TObject {
         this.(ObjectInternal).isBuiltin()
     }
 
-    /** Holds if this value represents an entity that is inferred to exist,
-     * but missing from the database.
-     * Most commonly, this is a module that is imported, but wasn't present during extraction.
-     */
-    predicate isMissing() {
-        this.(ObjectInternal).isMissing()
-    }
-
     predicate hasLocationInfo(string filepath, int bl, int bc, int el, int ec) {
         this.(ObjectInternal).getOrigin().getLocation().hasLocationInfo(filepath, bl, bc, el, ec)
         or

python/ql/src/semmle/python/objects/ObjectInternal.qll

@@ -155,11 +155,12 @@ class ObjectInternal extends TObject {
      */
     predicate functionAndOffset(CallableObjectInternal function, int offset) { none() }
 
-    /** Holds if this 'object' represents an entity that is inferred to exist
-     * but is missing from the database */
-    predicate isMissing() {
-        none()
-    }
+    /** Holds if this 'object' represents an entity that should be exposed to the legacy points_to API
+     * This should hold for almost all objects that do not have an underlying DB object representing their source,
+     * for example `super` objects and bound-method. This should not hold for objects that are inferred to exists by
+     * an import statements or the like, but which aren't in the database. */
+     /* This predicate can be removed when the legacy points_to API is removed. */
+    abstract predicate useOriginAsLegacyObject();
 
     /** Gets the name of this of this object if it has a meaningful name.
      * Note that the name of an object is not necessarily the name by which it is called
@@ -260,6 +261,8 @@ class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject {
 
     override predicate contextSensitiveCallee() { none() }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
     override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
 
 }
@@ -341,6 +344,8 @@ class UnknownInternal extends ObjectInternal, TUnknown {
 
     override predicate contextSensitiveCallee() { none() }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
     override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
 
 }
@@ -421,6 +426,8 @@ class UndefinedInternal extends ObjectInternal, TUndefined {
 
     override string getName() { none() }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
     /** Holds if this object requires context to determine the object resulting from a call to it.
      * True for most callables. */
     override predicate contextSensitiveCallee() { none() }
@@ -608,6 +615,8 @@ class DecoratedFunction extends ObjectInternal, TDecoratedFunction {
 
     override predicate contextSensitiveCallee() { none() }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
 }
 
 /** Helper for boolean predicates returning both `true` and `false` */

python/ql/src/semmle/python/objects/Sequences.qll

@@ -121,6 +121,9 @@ class BuiltinTupleObjectInternal extends TBuiltinTuple, TupleObjectInternal {
             result = count(int n | exists(b.getItem(n)))
         )
     }
+
+    override predicate useOriginAsLegacyObject() { none() }
+
 }
 
 /** A tuple declared by a tuple expression in the Python source code */
@@ -152,6 +155,8 @@ class PythonTupleObjectInternal extends TPythonTuple, TupleObjectInternal {
         )
     }
 
+    override predicate useOriginAsLegacyObject() { none() }
+
 }
 
 /** A tuple created by a `*` parameter */
@@ -180,6 +185,9 @@ class VarargsTupleObjectInternal extends TVarargsTuple,  TupleObjectInternal {
     override int length() {
         this = TVarargsTuple(_, _, _, result)
     }
+
+    override predicate useOriginAsLegacyObject() { any() }
+
 }
 
 
@@ -260,4 +268,6 @@ class SysVersionInfoObjectInternal extends TSysVersionInfo, SequenceObjectIntern
 
     override predicate functionAndOffset(CallableObjectInternal function, int offset) { none() }
 
+    override predicate useOriginAsLegacyObject() { any() }
+
 }

python/ql/src/semmle/python/pointsto/PointsTo.qll

@@ -130,7 +130,7 @@ module PointsTo {
             PointsToInternal::pointsTo(f, context, value, origin) and
             cls = value.getClass().getSource() |
             obj = value.getSource() or
-            not exists(value.getSource()) and not value.isMissing() and obj = origin
+            value.useOriginAsLegacyObject() and obj = origin
         )
         or
         /* Backwards compatibility for *args and **kwargs */
@@ -145,7 +145,7 @@ module PointsTo {
             PointsToInternal::pointsTo(f.(DefinitionNode).getValue(), context, value, origin) and
             cls = value.getClass().getSource() |
             obj = value.getSource() or
-            not exists(value.getSource()) and obj = origin
+            value.useOriginAsLegacyObject() and obj = origin
         )
     }
 

python/ql/src/semmle/python/regex.qll

@@ -1,4 +1,5 @@
 import python
+import semmle.python.objects.ObjectInternal
 
 private predicate re_module_function(string name, int flags) {
     name = "compile" and flags = 1 or
@@ -14,44 +15,42 @@ private predicate re_module_function(string name, int flags) {
 predicate used_as_regex(Expr s, string mode) {
     (s instanceof Bytes or s instanceof Unicode)
     and
-    exists(ModuleObject re | re.getName() = "re" |
+    exists(ModuleValue re | re.getName() = "re" |
         /* Call to re.xxx(regex, ... [mode]) */
         exists(CallNode call, string name |
             call.getArg(0).refersTo(_, _, s.getAFlowNode()) and
-            call.getFunction().refersTo(re.attr(name)) |
+            call.getFunction().pointsTo(re.attr(name)) |
             mode = "None"
             or
-            exists(Object obj |
+            exists(Value obj |
                 mode = mode_from_mode_object(obj) |
                 exists(int flags_arg |
                     re_module_function(name, flags_arg) and
-                    call.getArg(flags_arg).refersTo(obj)
+                    call.getArg(flags_arg).pointsTo(obj)
                 )
                 or
-                call.getArgByName("flags").refersTo(obj)
+                call.getArgByName("flags").pointsTo(obj)
             )
         )
     )
 }
 
-string mode_from_mode_object(Object obj) {
+string mode_from_mode_object(Value obj) {
     (
         result = "DEBUG" or result = "IGNORECASE" or result = "LOCALE" or
         result = "MULTILINE" or result = "DOTALL" or result = "UNICODE" or
         result = "VERBOSE"
     ) and
-    obj = ModuleObject::named("sre_constants").attr("SRE_FLAG_" + result)
-    or
-    exists(BinaryExpr be, Object sub | obj.getOrigin() = be |
-        be.getOp() instanceof BitOr and
-        be.getASubExpression().refersTo(sub) and
-        result = mode_from_mode_object(sub)
+    exists(int flag |
+        flag = Value::named("sre_constants.SRE_FLAG_" + result).(ObjectInternal).intValue()
+        and
+        obj.(ObjectInternal).intValue().bitAnd(flag) = flag
     )
 }
 
 /** A StrConst used as a regular expression */
 abstract class RegexString extends Expr {
-  
+
     RegexString() {
         (this instanceof Bytes or this instanceof Unicode)
     }

python/ql/src/semmle/python/types/Extensions.qll

@@ -13,9 +13,11 @@ import python
 private import semmle.python.pointsto.PointsTo
 private import semmle.python.pointsto.PointsToContext
 private import semmle.python.objects.TObject
-private import semmle.python.objects.ObjectInternal
 private import semmle.python.web.HttpConstants
 
+/* Make ObjectInternal visible to save extra imports in user code */
+import semmle.python.objects.ObjectInternal
+
 abstract class PointsToExtension extends @py_flow_node {
 
     string toString() { none() }