Merge pull request #1612 from markshannon/merge-121

Merge rc/1.21 into master
This commit is contained in:
Taus
2019-07-19 10:08:59 +01:00
committed by GitHub
20 changed files with 162 additions and 51 deletions

View File

@@ -33,6 +33,8 @@ import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.lang.ProcessBuilder.Redirect;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
/**
@@ -40,7 +42,12 @@ import java.util.List;
*
* <p>The Node.js half of the wrapper is expected to live at {@code
* $SEMMLE_DIST/tools/typescript-parser-wrapper/main.js}; non-standard locations can be configured
* using the property {@link #PARSER_WRAPPER_PATH_ENV_VAR}.
* using the property {@value #PARSER_WRAPPER_PATH_ENV_VAR}.
*
* <p>The script launches the Node.js wrapper in the Node.js runtime, looking for {@code node}
* on the {@code PATH} by default. Non-standard locations can be configured using the property
* {@value #TYPESCRIPT_NODE_RUNTIME_VAR}, and additional arguments can be configured using the
* property {@value #TYPESCRIPT_NODE_RUNTIME_EXTRA_ARGS_VAR}.
*
* <p>The script is started upon parsing the first TypeScript file and then is kept running in the
* background, passing it requests for parsing files and getting JSON-encoded ASTs as responses.
@@ -52,6 +59,18 @@ public class TypeScriptParser {
*/
public static final String PARSER_WRAPPER_PATH_ENV_VAR = "SEMMLE_TYPESCRIPT_PARSER_WRAPPER";
/**
* An environment variable that can be set to indicate the location of the Node.js runtime,
* as an alternative to adding Node to the PATH.
*/
public static final String TYPESCRIPT_NODE_RUNTIME_VAR = "SEMMLE_TYPESCRIPT_NODE_RUNTIME";
/**
* An environment variable that can be set to provide additional arguments to the Node.js runtime
* each time it is invoked. Arguments should be separated by spaces.
*/
public static final String TYPESCRIPT_NODE_RUNTIME_EXTRA_ARGS_VAR = "SEMMLE_TYPESCRIPT_NODE_RUNTIME_EXTRA_ARGS";
/**
* An environment variable that can be set to specify a timeout to use when verifying the
* TypeScript installation, in milliseconds. Default is 10000.
@@ -91,6 +110,15 @@ public class TypeScriptParser {
private String nodeJsVersionString;
/** Command to launch the Node.js runtime. Initialised by {@link #verifyNodeInstallation}. */
private String nodeJsRuntime;
/**
* Arguments to pass to the Node.js runtime each time it is invoked.
* Initialised by {@link #verifyNodeInstallation}.
*/
private List<String> nodeJsRuntimeExtraArgs = Collections.emptyList();
/** If non-zero, we use this instead of relying on the corresponding environment variable. */
private int typescriptRam = 0;
@@ -102,12 +130,16 @@ public class TypeScriptParser {
/**
* Verifies that Node.js and TypeScript are installed and throws an exception otherwise.
*
* @param verbose if true, log the version strings and NODE_PATH.
* @param verbose if true, log the Node.js executable path, version strings, and any additional arguments.
*/
public void verifyInstallation(boolean verbose) {
verifyNodeInstallation();
if (verbose) {
System.out.println("Found Node.js at: " + nodeJsRuntime);
System.out.println("Found Node.js version: " + nodeJsVersionString);
if (!nodeJsRuntimeExtraArgs.isEmpty()) {
System.out.println("Additional arguments for Node.js: " + nodeJsRuntimeExtraArgs);
}
}
}
@@ -117,7 +149,24 @@ public class TypeScriptParser {
ByteArrayOutputStream out = new ByteArrayOutputStream();
ByteArrayOutputStream err = new ByteArrayOutputStream();
Builder b = new Builder(out, err, getParserWrapper().getParentFile(), "node", "--version");
// Determine where to find the Node.js runtime.
String explicitNodeJsRuntime = Env.systemEnv().get(TYPESCRIPT_NODE_RUNTIME_VAR);
if (explicitNodeJsRuntime != null) {
// Use the specified Node.js executable.
nodeJsRuntime = explicitNodeJsRuntime;
} else {
// Look for `node` on the PATH.
nodeJsRuntime = "node";
}
// Determine any additional arguments to be passed to Node.js each time it's called.
String extraArgs = Env.systemEnv().get(TYPESCRIPT_NODE_RUNTIME_EXTRA_ARGS_VAR);
if (extraArgs != null) {
nodeJsRuntimeExtraArgs = Arrays.asList(extraArgs.split("\\s+"));
}
Builder b = new Builder(getNodeJsRuntimeInvocation("--version"), out, err, getParserWrapper().getParentFile());
b.expectFailure(); // We want to do our own logging in case of an error.
int timeout = Env.systemEnv().getInt(TYPESCRIPT_TIMEOUT_VAR, 10000);
@@ -144,6 +193,21 @@ public class TypeScriptParser {
}
}
/**
* Gets a command line to invoke the Node.js runtime.
* Any arguments in {@link TypeScriptParser#nodeJsRuntimeExtraArgs}
* are passed first, followed by those in {@code args}.
*/
private List<String> getNodeJsRuntimeInvocation(String ...args) {
List<String> result = new ArrayList<>();
result.add(nodeJsRuntime);
result.addAll(nodeJsRuntimeExtraArgs);
for(String arg : args) {
result.add(arg);
}
return result;
}
private static int getMegabyteCountFromPrefixedEnv(String suffix, int defaultValue) {
String envVar = "SEMMLE_" + suffix;
String value = Env.systemEnv().get(envVar);
@@ -172,10 +236,11 @@ public class TypeScriptParser {
int reserveMemoryMb = getMegabyteCountFromPrefixedEnv(TYPESCRIPT_RAM_RESERVE_SUFFIX, 400);
File parserWrapper = getParserWrapper();
List<String> cmd = new ArrayList<>();
cmd.add("node");
cmd.add("--max_old_space_size=" + (mainMemoryMb + reserveMemoryMb));
cmd.add(parserWrapper.getAbsolutePath());
List<String> cmd = getNodeJsRuntimeInvocation(
"--max_old_space_size=" + (mainMemoryMb + reserveMemoryMb),
parserWrapper.getAbsolutePath()
);
ProcessBuilder pb = new ProcessBuilder(cmd);
parserWrapperCommand = StringUtil.glue(" ", cmd);
pb.environment().put("SEMMLE_TYPESCRIPT_MEMORY_THRESHOLD", "" + mainMemoryMb);

View File

@@ -161,6 +161,8 @@ class PythonFunctionObjectInternal extends CallableObjectInternal, TPythonFuncti
override predicate contextSensitiveCallee() { any() }
override predicate useOriginAsLegacyObject() { none() }
}
@@ -284,6 +286,8 @@ class BuiltinFunctionObjectInternal extends CallableObjectInternal, TBuiltinFunc
override predicate contextSensitiveCallee() { none() }
override predicate useOriginAsLegacyObject() { none() }
}
/** Class representing methods of built-in classes (otherwise known as method-descriptors) such as `list.append`.
@@ -376,6 +380,8 @@ class BuiltinMethodObjectInternal extends CallableObjectInternal, TBuiltinMethod
override predicate contextSensitiveCallee() { none() }
override predicate useOriginAsLegacyObject() { none() }
}
/** Class representing bound-methods.
@@ -461,6 +467,8 @@ class BoundMethodObjectInternal extends CallableObjectInternal, TBoundMethod {
function = this.getFunction() and offset = 1
}
override predicate useOriginAsLegacyObject() { any() }
override predicate contextSensitiveCallee() {
this.getFunction().contextSensitiveCallee()
}

View File

@@ -92,6 +92,8 @@ abstract class ClassObjectInternal extends ObjectInternal {
override predicate contextSensitiveCallee() { none() }
override predicate useOriginAsLegacyObject() { none() }
/* Classes aren't usually iterable, but can e.g. Enums */
override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }

View File

@@ -71,6 +71,8 @@ abstract class ConstantObjectInternal extends ObjectInternal {
override predicate contextSensitiveCallee() { none() }
override predicate useOriginAsLegacyObject() { none() }
}
private abstract class BooleanObjectInternal extends ConstantObjectInternal {

View File

@@ -93,6 +93,8 @@ class PropertyInternal extends ObjectInternal, TProperty {
override predicate contextSensitiveCallee() { none() }
override predicate useOriginAsLegacyObject() { none() }
/* Properties aren't iterable */
override ObjectInternal getIterNext() { none() }
@@ -183,6 +185,8 @@ class ClassMethodObjectInternal extends ObjectInternal, TClassMethod {
override predicate contextSensitiveCallee() { none() }
override predicate useOriginAsLegacyObject() { none() }
/* Classmethods aren't iterable */
override ObjectInternal getIterNext() { none() }
@@ -259,6 +263,8 @@ class StaticMethodObjectInternal extends ObjectInternal, TStaticMethod {
override predicate contextSensitiveCallee() { none() }
override predicate useOriginAsLegacyObject() { none() }
/* Staticmethods aren't iterable */
override ObjectInternal getIterNext() { none() }

View File

@@ -164,6 +164,8 @@ class SpecificInstanceInternal extends TSpecificInstance, InstanceObject {
)
}
override predicate useOriginAsLegacyObject() { none() }
}
/** A class representing context-free instances represented by `self` in the source code
@@ -266,6 +268,8 @@ class SelfInstanceInternal extends TSelfInstance, InstanceObject {
this.getClass().attribute("__init__", init, _)
}
override predicate useOriginAsLegacyObject() { none() }
}
/** A class representing a value that has a known class, but no other information */
@@ -372,6 +376,8 @@ class UnknownInstanceInternal extends TUnknownInstance, ObjectInternal {
override predicate contextSensitiveCallee() { none() }
override predicate useOriginAsLegacyObject() { any() }
override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
}
@@ -482,6 +488,8 @@ class SuperInstance extends TSuperInstance, ObjectInternal {
override predicate contextSensitiveCallee() { none() }
override predicate useOriginAsLegacyObject() { any() }
override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
}

View File

@@ -54,6 +54,8 @@ abstract class ModuleObjectInternal extends ObjectInternal {
override predicate contextSensitiveCallee() { none() }
override predicate useOriginAsLegacyObject() { none() }
/* Modules aren't iterable */
override ObjectInternal getIterNext() { none() }
@@ -313,10 +315,6 @@ class AbsentModuleObjectInternal extends ModuleObjectInternal, TAbsentModule {
none()
}
override predicate isMissing() {
any()
}
}
/** A class representing an attribute of a missing module. */
@@ -402,15 +400,13 @@ class AbsentModuleAttributeObjectInternal extends ObjectInternal, TAbsentModuleA
override predicate subscriptUnknown() { any() }
override predicate isMissing() {
any()
}
/* We know what this is called, but not its innate name */
override string getName() { none() }
override predicate contextSensitiveCallee() { none() }
override predicate useOriginAsLegacyObject() { none() }
/* Modules aren't iterable */
override ObjectInternal getIterNext() { none() }

View File

@@ -78,14 +78,6 @@ class Value extends TObject {
this.(ObjectInternal).isBuiltin()
}
/** Holds if this value represents an entity that is inferred to exist,
* but missing from the database.
* Most commonly, this is a module that is imported, but wasn't present during extraction.
*/
predicate isMissing() {
this.(ObjectInternal).isMissing()
}
predicate hasLocationInfo(string filepath, int bl, int bc, int el, int ec) {
this.(ObjectInternal).getOrigin().getLocation().hasLocationInfo(filepath, bl, bc, el, ec)
or

View File

@@ -155,11 +155,12 @@ class ObjectInternal extends TObject {
*/
predicate functionAndOffset(CallableObjectInternal function, int offset) { none() }
/** Holds if this 'object' represents an entity that is inferred to exist
* but is missing from the database */
predicate isMissing() {
none()
}
/** Holds if this 'object' represents an entity that should be exposed to the legacy points_to API
* This should hold for almost all objects that do not have an underlying DB object representing their source,
* for example `super` objects and bound-method. This should not hold for objects that are inferred to exists by
* an import statements or the like, but which aren't in the database. */
/* This predicate can be removed when the legacy points_to API is removed. */
abstract predicate useOriginAsLegacyObject();
/** Gets the name of this of this object if it has a meaningful name.
* Note that the name of an object is not necessarily the name by which it is called
@@ -260,6 +261,8 @@ class BuiltinOpaqueObjectInternal extends ObjectInternal, TBuiltinOpaqueObject {
override predicate contextSensitiveCallee() { none() }
override predicate useOriginAsLegacyObject() { none() }
override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
}
@@ -341,6 +344,8 @@ class UnknownInternal extends ObjectInternal, TUnknown {
override predicate contextSensitiveCallee() { none() }
override predicate useOriginAsLegacyObject() { none() }
override ObjectInternal getIterNext() { result = ObjectInternal::unknown() }
}
@@ -421,6 +426,8 @@ class UndefinedInternal extends ObjectInternal, TUndefined {
override string getName() { none() }
override predicate useOriginAsLegacyObject() { none() }
/** Holds if this object requires context to determine the object resulting from a call to it.
* True for most callables. */
override predicate contextSensitiveCallee() { none() }
@@ -608,6 +615,8 @@ class DecoratedFunction extends ObjectInternal, TDecoratedFunction {
override predicate contextSensitiveCallee() { none() }
override predicate useOriginAsLegacyObject() { none() }
}
/** Helper for boolean predicates returning both `true` and `false` */

View File

@@ -121,6 +121,9 @@ class BuiltinTupleObjectInternal extends TBuiltinTuple, TupleObjectInternal {
result = count(int n | exists(b.getItem(n)))
)
}
override predicate useOriginAsLegacyObject() { none() }
}
/** A tuple declared by a tuple expression in the Python source code */
@@ -152,6 +155,8 @@ class PythonTupleObjectInternal extends TPythonTuple, TupleObjectInternal {
)
}
override predicate useOriginAsLegacyObject() { none() }
}
/** A tuple created by a `*` parameter */
@@ -180,6 +185,9 @@ class VarargsTupleObjectInternal extends TVarargsTuple, TupleObjectInternal {
override int length() {
this = TVarargsTuple(_, _, _, result)
}
override predicate useOriginAsLegacyObject() { any() }
}
@@ -260,4 +268,6 @@ class SysVersionInfoObjectInternal extends TSysVersionInfo, SequenceObjectIntern
override predicate functionAndOffset(CallableObjectInternal function, int offset) { none() }
override predicate useOriginAsLegacyObject() { any() }
}

View File

@@ -130,7 +130,7 @@ module PointsTo {
PointsToInternal::pointsTo(f, context, value, origin) and
cls = value.getClass().getSource() |
obj = value.getSource() or
not exists(value.getSource()) and not value.isMissing() and obj = origin
value.useOriginAsLegacyObject() and obj = origin
)
or
/* Backwards compatibility for *args and **kwargs */
@@ -145,7 +145,7 @@ module PointsTo {
PointsToInternal::pointsTo(f.(DefinitionNode).getValue(), context, value, origin) and
cls = value.getClass().getSource() |
obj = value.getSource() or
not exists(value.getSource()) and obj = origin
value.useOriginAsLegacyObject() and obj = origin
)
}

View File

@@ -1,4 +1,5 @@
import python
import semmle.python.objects.ObjectInternal
private predicate re_module_function(string name, int flags) {
name = "compile" and flags = 1 or
@@ -14,44 +15,42 @@ private predicate re_module_function(string name, int flags) {
predicate used_as_regex(Expr s, string mode) {
(s instanceof Bytes or s instanceof Unicode)
and
exists(ModuleObject re | re.getName() = "re" |
exists(ModuleValue re | re.getName() = "re" |
/* Call to re.xxx(regex, ... [mode]) */
exists(CallNode call, string name |
call.getArg(0).refersTo(_, _, s.getAFlowNode()) and
call.getFunction().refersTo(re.attr(name)) |
call.getFunction().pointsTo(re.attr(name)) |
mode = "None"
or
exists(Object obj |
exists(Value obj |
mode = mode_from_mode_object(obj) |
exists(int flags_arg |
re_module_function(name, flags_arg) and
call.getArg(flags_arg).refersTo(obj)
call.getArg(flags_arg).pointsTo(obj)
)
or
call.getArgByName("flags").refersTo(obj)
call.getArgByName("flags").pointsTo(obj)
)
)
)
}
string mode_from_mode_object(Object obj) {
string mode_from_mode_object(Value obj) {
(
result = "DEBUG" or result = "IGNORECASE" or result = "LOCALE" or
result = "MULTILINE" or result = "DOTALL" or result = "UNICODE" or
result = "VERBOSE"
) and
obj = ModuleObject::named("sre_constants").attr("SRE_FLAG_" + result)
or
exists(BinaryExpr be, Object sub | obj.getOrigin() = be |
be.getOp() instanceof BitOr and
be.getASubExpression().refersTo(sub) and
result = mode_from_mode_object(sub)
exists(int flag |
flag = Value::named("sre_constants.SRE_FLAG_" + result).(ObjectInternal).intValue()
and
obj.(ObjectInternal).intValue().bitAnd(flag) = flag
)
}
/** A StrConst used as a regular expression */
abstract class RegexString extends Expr {
RegexString() {
(this instanceof Bytes or this instanceof Unicode)
}

View File

@@ -13,9 +13,11 @@ import python
private import semmle.python.pointsto.PointsTo
private import semmle.python.pointsto.PointsToContext
private import semmle.python.objects.TObject
private import semmle.python.objects.ObjectInternal
private import semmle.python.web.HttpConstants
/* Make ObjectInternal visible to save extra imports in user code */
import semmle.python.objects.ObjectInternal
abstract class PointsToExtension extends @py_flow_node {
string toString() { none() }

View File

@@ -1,4 +1,3 @@
| 1 | ControlFlowNode for unicode_literals | ImportMember | 1 |
| 2 | ControlFlowNode for C | class C | 2 |
| 2 | ControlFlowNode for ClassExpr | class C | 2 |
| 2 | ControlFlowNode for object | builtin-class object | 2 |

View File

@@ -1,6 +1,8 @@
WARNING: Type CustomPointsToAttribute has been deprecated and may be removed in future (Extend.ql:26,35-58)
WARNING: Type CustomPointsToObjectFact has been deprecated and may be removed in future (Extend.ql:41,32-56)
WARNING: Type CustomPointsToOriginFact has been deprecated and may be removed in future (Extend.ql:8,28-52)
WARNING: Predicate points_to has been deprecated and may be removed in future (Extend.ql:58,9-28)
WARNING: Type CustomPointsToAttribute has been deprecated and may be removed in future (Extend.ql:27,35-58)
WARNING: Type CustomPointsToObjectFact has been deprecated and may be removed in future (Extend.ql:42,32-56)
WARNING: Type CustomPointsToOriginFact has been deprecated and may be removed in future (Extend.ql:9,28-52)
WARNING: Type CustomPointsToOriginFact has been deprecated and may be removed in future (Extend.ql:55,38-62)
| test.py:4:1:4:3 | ControlFlowNode for one | int 1 |
| test.py:5:1:5:3 | ControlFlowNode for two | int 2 |
| test.py:8:1:8:1 | ControlFlowNode for IntegerLiteral | int 1 |

View File

@@ -2,6 +2,7 @@
import python
import semmle.python.pointsto.PointsTo
private import semmle.python.types.Extensions
@@ -50,6 +51,18 @@ class NoClassExtension extends CustomPointsToObjectFact {
}
/* Check that we can use old API without causing non-monotonic recursion */
class RecurseIntoOldPointsTo extends CustomPointsToOriginFact {
RecurseIntoOldPointsTo() {
PointsTo::points_to(this, _, unknownValue(), _, _)
}
override predicate pointsTo(Object value, ClassObject cls) {
value = unknownValue() and cls = theUnknownType()
}
}
from ControlFlowNode f, Object o
where f.getLocation().getFile().getBaseName() = "test.py" and f.refersTo(o)

View File

@@ -98,7 +98,6 @@
| Module pointsto_test | 76 | ControlFlowNode for sys | Module sys |
| Module pointsto_test | 76 | ControlFlowNode for type | builtin-class type |
| Module pointsto_test | 76 | ControlFlowNode for type() | builtin-class module |
| Module pointsto_test | 77 | ControlFlowNode for unknown | ImportMember |
| Module pointsto_test | 78 | ControlFlowNode for type | builtin-class type |
| Module pointsto_test | 79 | ControlFlowNode for Dict | Dict |
| Module pointsto_test | 79 | ControlFlowNode for Tuple | Tuple |

View File

@@ -106,7 +106,6 @@
| 76 | ControlFlowNode for sys | Module sys |
| 76 | ControlFlowNode for type | builtin-class type |
| 76 | ControlFlowNode for type() | builtin-class module |
| 77 | ControlFlowNode for unknown | ImportMember |
| 78 | ControlFlowNode for type | builtin-class type |
| 79 | ControlFlowNode for Dict | Dict |
| 79 | ControlFlowNode for Tuple | Tuple |

View File

@@ -1170,7 +1170,6 @@ WARNING: Predicate points_to has been deprecated and may be removed in future (P
| t_type.py:7 | ControlFlowNode for sys | Module sys | builtin-class module | 1 | import |
| t_type.py:7 | ControlFlowNode for type | builtin-class type | builtin-class type | 7 | import |
| t_type.py:7 | ControlFlowNode for type() | builtin-class module | builtin-class type | 7 | import |
| t_type.py:8 | ControlFlowNode for unknown | ImportMember | *UNKNOWN TYPE* | 8 | import |
| t_type.py:9 | ControlFlowNode for type | builtin-class type | builtin-class type | 9 | import |
| t_type.py:9 | ControlFlowNode for type() | *UNKNOWN TYPE* | *UNKNOWN TYPE* | 9 | import |
| t_type.py:10 | ControlFlowNode for Dict | Dict | builtin-class dict | 10 | import |

View File

@@ -1 +1,2 @@
semmle-extractor-options: --max-import-depth=3
optimize: true