diff --git a/python/ql/src/default.qll b/python/ql/src/default.qll index 99374e13f76..8fa398db399 100644 --- a/python/ql/src/default.qll +++ b/python/ql/src/default.qll @@ -2,4 +2,5 @@ * WARNING: Use of this module is DEPRECATED. * All new queries should use `import python`. */ + import python diff --git a/python/ql/src/external/CodeDuplication.qll b/python/ql/src/external/CodeDuplication.qll index 3178f8d463a..c195d16ae6a 100644 --- a/python/ql/src/external/CodeDuplication.qll +++ b/python/ql/src/external/CodeDuplication.qll @@ -3,10 +3,7 @@ import python /** Gets the relative path of `file`, with backslashes replaced by forward slashes. */ -private -string relativePath(File file) { - result = file.getRelativePath().replaceAll("\\", "/") -} +private string relativePath(File file) { result = file.getRelativePath().replaceAll("\\", "/") } /** * Holds if the `index`-th token of block `copy` is in file `file`, spanning @@ -21,60 +18,42 @@ private predicate tokenLocation(File file, int sl, int sc, int ec, int el, Copy } /** A token block used for detection of duplicate and similar code. */ -class Copy extends @duplication_or_similarity -{ - private - int lastToken() { - result = max(int i | tokens(this, i, _, _, _, _) | i) - } +class Copy extends @duplication_or_similarity { + private int lastToken() { result = max(int i | tokens(this, i, _, _, _, _) | i) } /** Gets the index of the token in this block starting at the location `loc`, if any. */ int tokenStartingAt(Location loc) { - tokenLocation(loc.getFile(), loc.getStartLine(), loc.getStartColumn(), - _, _, this, result) + tokenLocation(loc.getFile(), loc.getStartLine(), loc.getStartColumn(), _, _, this, result) } /** Gets the index of the token in this block ending at the location `loc`, if any. */ int tokenEndingAt(Location loc) { - tokenLocation(loc.getFile(), _, _, - loc.getEndLine(), loc.getEndColumn(), this, result) + tokenLocation(loc.getFile(), _, _, loc.getEndLine(), loc.getEndColumn(), this, result) } /** Gets the line on which the first token in this block starts. */ - int sourceStartLine() { - tokens(this, 0, result, _, _, _) - } + int sourceStartLine() { tokens(this, 0, result, _, _, _) } /** Gets the column on which the first token in this block starts. */ - int sourceStartColumn() { - tokens(this, 0, _, result, _, _) - } + int sourceStartColumn() { tokens(this, 0, _, result, _, _) } /** Gets the line on which the last token in this block ends. */ - int sourceEndLine() { - tokens(this, this.lastToken(), _, _, result, _) - } + int sourceEndLine() { tokens(this, this.lastToken(), _, _, result, _) } /** Gets the column on which the last token in this block ends. */ - int sourceEndColumn() { - tokens(this, this.lastToken(), _, _, _, result) - } + int sourceEndColumn() { tokens(this, this.lastToken(), _, _, _, result) } /** Gets the number of lines containing at least (part of) one token in this block. */ - int sourceLines() { - result = this.sourceEndLine() + 1 - this.sourceStartLine() - } + int sourceLines() { result = this.sourceEndLine() + 1 - this.sourceStartLine() } /** Gets an opaque identifier for the equivalence class of this block. */ - int getEquivalenceClass() { - duplicateCode(this, _, result) or similarCode(this, _, result) - } + int getEquivalenceClass() { duplicateCode(this, _, result) or similarCode(this, _, result) } /** Gets the source file in which this block appears. */ File sourceFile() { - exists(string name | - duplicateCode(this, name, _) or similarCode(this, name, _) | - name.replaceAll("\\", "/") = relativePath(result)) + exists(string name | duplicateCode(this, name, _) or similarCode(this, name, _) | + name.replaceAll("\\", "/") = relativePath(result) + ) } /** @@ -84,7 +63,9 @@ class Copy extends @duplication_or_similarity * For more information, see * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html). */ - predicate hasLocationInfo(string filepath, int startline, int startcolumn, int endline, int endcolumn) { + predicate hasLocationInfo( + string filepath, int startline, int startcolumn, int endline, int endcolumn + ) { sourceFile().getName() = filepath and startline = sourceStartLine() and startcolumn = sourceStartColumn() and @@ -102,7 +83,8 @@ class Copy extends @duplication_or_similarity Copy extendingBlock() { exists(File file, int sl, int sc, int ec, int el | tokenLocation(file, sl, sc, ec, el, this, _) and - tokenLocation(file, sl, sc, ec, el, result, 0)) and + tokenLocation(file, sl, sc, ec, el, result, 0) + ) and this != result } } @@ -113,13 +95,17 @@ class Copy extends @duplication_or_similarity * have the same equivalence class, with `start` being the equivalence class of `start1` and * `start2`, and `end` the equivalence class of `end1` and `end2`. */ -predicate similar_extension(SimilarBlock start1, SimilarBlock start2, SimilarBlock ext1, SimilarBlock ext2, int start, int ext) { +predicate similar_extension( + SimilarBlock start1, SimilarBlock start2, SimilarBlock ext1, SimilarBlock ext2, int start, int ext +) { start1.getEquivalenceClass() = start and start2.getEquivalenceClass() = start and ext1.getEquivalenceClass() = ext and ext2.getEquivalenceClass() = ext and start1 != start2 and - (ext1 = start1 and ext2 = start2 or + ( + ext1 = start1 and ext2 = start2 + or similar_extension(start1.extendingBlock(), start2.extendingBlock(), ext1, ext2, _, ext) ) } @@ -130,28 +116,29 @@ predicate similar_extension(SimilarBlock start1, SimilarBlock start2, SimilarBlo * have the same equivalence class, with `start` being the equivalence class of `start1` and * `start2`, and `end` the equivalence class of `end1` and `end2`. */ -predicate duplicate_extension(DuplicateBlock start1, DuplicateBlock start2, DuplicateBlock ext1, DuplicateBlock ext2, int start, int ext) { +predicate duplicate_extension( + DuplicateBlock start1, DuplicateBlock start2, DuplicateBlock ext1, DuplicateBlock ext2, int start, + int ext +) { start1.getEquivalenceClass() = start and start2.getEquivalenceClass() = start and ext1.getEquivalenceClass() = ext and ext2.getEquivalenceClass() = ext and start1 != start2 and - (ext1 = start1 and ext2 = start2 or + ( + ext1 = start1 and ext2 = start2 + or duplicate_extension(start1.extendingBlock(), start2.extendingBlock(), ext1, ext2, _, ext) ) } /** A block of duplicated code. */ -class DuplicateBlock extends Copy, @duplication -{ - override string toString() { - result = "Duplicate code: " + sourceLines() + " duplicated lines." - } +class DuplicateBlock extends Copy, @duplication { + override string toString() { result = "Duplicate code: " + sourceLines() + " duplicated lines." } } /** A block of similar code. */ -class SimilarBlock extends Copy, @similarity -{ +class SimilarBlock extends Copy, @similarity { override string toString() { result = "Similar code: " + sourceLines() + " almost duplicated lines." } @@ -167,8 +154,9 @@ predicate duplicateStatement(Scope scope1, Scope scope2, Stmt stmt1, Stmt stmt2) scope2.contains(stmt2) and duplicateCoversStatement(equivstart, equivend, first, last, stmt1) and duplicateCoversStatement(equivstart, equivend, first, last, stmt2) and - stmt1 != stmt2 and scope1 != scope2 - ) + stmt1 != stmt2 and + scope1 != scope2 + ) } /** @@ -178,8 +166,9 @@ predicate duplicateStatement(Scope scope1, Scope scope2, Stmt stmt1, Stmt stmt2) * and `equivstart` and `equivend` are the equivalence classes of the first and the last * block, respectively. */ -private -predicate duplicateCoversStatement(int equivstart, int equivend, int first, int last, Stmt stmt) { +private predicate duplicateCoversStatement( + int equivstart, int equivend, int first, int last, Stmt stmt +) { exists(DuplicateBlock b1, DuplicateBlock b2, Location startloc, Location endloc | stmt.getLocation() = startloc and stmt.getLastStatement().getLocation() = endloc and @@ -204,13 +193,14 @@ predicate duplicateStatements(Scope scope1, Scope scope2, int duplicate, int tot * Find pairs of scopes that are identical or almost identical */ predicate duplicateScopes(Scope s, Scope other, float percent, string message) { - exists(int total, int duplicate | - duplicateStatements(s, other, duplicate, total) | - percent = 100.0 * duplicate / total and percent >= 80.0 and - if duplicate = total then - message = "All " + total + " statements in " + s.getName() + " are identical in $@." + exists(int total, int duplicate | duplicateStatements(s, other, duplicate, total) | + percent = 100.0 * duplicate / total and + percent >= 80.0 and + if duplicate = total + then message = "All " + total + " statements in " + s.getName() + " are identical in $@." else - message = duplicate + " out of " + total + " statements in " + s.getName() + " are duplicated in $@." + message = + duplicate + " out of " + total + " statements in " + s.getName() + " are duplicated in $@." ) } @@ -219,12 +209,13 @@ predicate duplicateScopes(Scope s, Scope other, float percent, string message) { * respectively, where `scope1` and `scope2` are not the same. */ private predicate similarStatement(Scope scope1, Scope scope2, Stmt stmt1, Stmt stmt2) { - exists(int start, int end, int first, int last | + exists(int start, int end, int first, int last | scope1.contains(stmt1) and scope2.contains(stmt2) and similarCoversStatement(start, end, first, last, stmt1) and similarCoversStatement(start, end, first, last, stmt2) and - stmt1 != stmt2 and scope1 != scope2 + stmt1 != stmt2 and + scope1 != scope2 ) } @@ -235,7 +226,9 @@ private predicate similarStatement(Scope scope1, Scope scope2, Stmt stmt1, Stmt * and `equivstart` and `equivend` are the equivalence classes of the first and the last * block, respectively. */ -private predicate similarCoversStatement(int equivstart, int equivend, int first, int last, Stmt stmt) { +private predicate similarCoversStatement( + int equivstart, int equivend, int first, int last, Stmt stmt +) { exists(SimilarBlock b1, SimilarBlock b2, Location startloc, Location endloc | stmt.getLocation() = startloc and stmt.getLastStatement().getLocation() = endloc and @@ -260,13 +253,14 @@ private predicate similarStatements(Scope scope1, Scope scope2, int similar, int * Find pairs of scopes that are similar */ predicate similarScopes(Scope s, Scope other, float percent, string message) { - exists(int total, int similar | - similarStatements(s, other, similar, total) | - percent = 100.0 * similar / total and percent >= 80.0 and - if similar = total then - message = "All statements in " + s.getName() + " are similar in $@." + exists(int total, int similar | similarStatements(s, other, similar, total) | + percent = 100.0 * similar / total and + percent >= 80.0 and + if similar = total + then message = "All statements in " + s.getName() + " are similar in $@." else - message = similar + " out of " + total + " statements in " + s.getName() + " are similar in $@." + message = + similar + " out of " + total + " statements in " + s.getName() + " are similar in $@." ) } @@ -275,7 +269,5 @@ predicate similarScopes(Scope s, Scope other, float percent, string message) { * This is true for blocks of import statements. */ predicate whitelistedLineForDuplication(File f, int line) { - exists(ImportingStmt i | - i.getLocation().getFile() = f and i.getLocation().getStartLine() = line - ) + exists(ImportingStmt i | i.getLocation().getFile() = f and i.getLocation().getStartLine() = line) } diff --git a/python/ql/src/external/DefectFilter.qll b/python/ql/src/external/DefectFilter.qll index e34385acfb8..6d5f75510cf 100644 --- a/python/ql/src/external/DefectFilter.qll +++ b/python/ql/src/external/DefectFilter.qll @@ -10,58 +10,61 @@ import semmle.python.Files * * For more information, see [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html). */ -external predicate defectResults(int id, string queryPath, string filepath, int startline, - int startcol, int endline, int endcol, string message); +external predicate defectResults( + int id, string queryPath, string filepath, int startline, int startcol, int endline, int endcol, + string message +); /** * A defect query result stored in a dashboard database. */ class DefectResult extends int { + DefectResult() { defectResults(this, _, _, _, _, _, _, _) } - DefectResult() { defectResults(this, _, _, _, _, _, _, _) } + /** Gets the path of the query that reported the result. */ + string getQueryPath() { defectResults(this, result, _, _, _, _, _, _) } - /** Gets the path of the query that reported the result. */ - string getQueryPath() { defectResults(this, result, _, _, _, _, _, _) } + /** Gets the file in which this query result was reported. */ + File getFile() { + exists(string path | defectResults(this, _, path, _, _, _, _, _) and result.getName() = path) + } - /** Gets the file in which this query result was reported. */ - File getFile() { - exists(string path | defectResults(this, _, path, _, _, _, _, _) and result.getName() = path) - } + /** Gets the file path in which this query result was reported. */ + string getFilePath() { defectResults(this, _, result, _, _, _, _, _) } - /** Gets the file path in which this query result was reported. */ - string getFilePath() { defectResults(this, _, result, _, _, _, _, _) } + /** Gets the line on which the location of this query result starts. */ + int getStartLine() { defectResults(this, _, _, result, _, _, _, _) } - /** Gets the line on which the location of this query result starts. */ - int getStartLine() { defectResults(this, _, _, result, _, _, _, _) } + /** Gets the column on which the location of this query result starts. */ + int getStartColumn() { defectResults(this, _, _, _, result, _, _, _) } - /** Gets the column on which the location of this query result starts. */ - int getStartColumn() { defectResults(this, _, _, _, result, _, _, _) } + /** Gets the line on which the location of this query result ends. */ + int getEndLine() { defectResults(this, _, _, _, _, result, _, _) } - /** Gets the line on which the location of this query result ends. */ - int getEndLine() { defectResults(this, _, _, _, _, result, _, _) } + /** Gets the column on which the location of this query result ends. */ + int getEndColumn() { defectResults(this, _, _, _, _, _, result, _) } - /** Gets the column on which the location of this query result ends. */ - int getEndColumn() { defectResults(this, _, _, _, _, _, result, _) } + /** Gets the message associated with this query result. */ + string getMessage() { defectResults(this, _, _, _, _, _, _, result) } - /** Gets the message associated with this query result. */ - string getMessage() { defectResults(this, _, _, _, _, _, _, result) } - - predicate hasLocationInfo(string path, int sl, int sc, int el, int ec) { - defectResults(this, _, path, sl, sc, el, ec, _) - } - - /** Gets the URL corresponding to the location of this query result. */ - string getURL() { - result = "file://" + getFile().getName() + ":" + getStartLine() + ":" + getStartColumn() + ":" + getEndLine() + ":" + getEndColumn() - } + predicate hasLocationInfo(string path, int sl, int sc, int el, int ec) { + defectResults(this, _, path, sl, sc, el, ec, _) + } + /** Gets the URL corresponding to the location of this query result. */ + string getURL() { + result = + "file://" + getFile().getName() + ":" + getStartLine() + ":" + getStartColumn() + ":" + + getEndLine() + ":" + getEndColumn() + } } // crude containment by line number only predicate contains(Location l, DefectResult res) { - exists(string path, int bl1, int el1, int bl2, int el2 | - l.hasLocationInfo(path, bl1, _, el1, _) - and res.hasLocationInfo(path, bl2, _, el2, _) - and bl1 <= bl2 and el1 >= el2 - ) + exists(string path, int bl1, int el1, int bl2, int el2 | + l.hasLocationInfo(path, bl1, _, el1, _) and + res.hasLocationInfo(path, bl2, _, el2, _) and + bl1 <= bl2 and + el1 >= el2 + ) } diff --git a/python/ql/src/external/ExternalArtifact.qll b/python/ql/src/external/ExternalArtifact.qll index 9c2fe6a9b66..0ed2c8d4878 100644 --- a/python/ql/src/external/ExternalArtifact.qll +++ b/python/ql/src/external/ExternalArtifact.qll @@ -1,7 +1,6 @@ import python class ExternalDefect extends @externalDefect { - string getQueryPath() { exists(string path | externalDefects(this, path, _, _, _) and @@ -9,95 +8,59 @@ class ExternalDefect extends @externalDefect { ) } - string getMessage() { - externalDefects(this, _, _, result, _) - } + string getMessage() { externalDefects(this, _, _, result, _) } - float getSeverity() { - externalDefects(this, _, _, _, result) - } + float getSeverity() { externalDefects(this, _, _, _, result) } - Location getLocation() { - externalDefects(this,_,result,_,_) - } + Location getLocation() { externalDefects(this, _, result, _, _) } - string toString() { - result = getQueryPath() + ": " + getLocation() + " - " + getMessage() - } + string toString() { result = getQueryPath() + ": " + getLocation() + " - " + getMessage() } } class ExternalMetric extends @externalMetric { + string getQueryPath() { externalMetrics(this, result, _, _) } - string getQueryPath() { - externalMetrics(this, result, _, _) - } + float getValue() { externalMetrics(this, _, _, result) } - float getValue() { - externalMetrics(this, _, _, result) - } + Location getLocation() { externalMetrics(this, _, result, _) } - Location getLocation() { - externalMetrics(this,_,result,_) - } - - string toString() { - result = getQueryPath() + ": " + getLocation() + " - " + getValue() - } + string toString() { result = getQueryPath() + ": " + getLocation() + " - " + getValue() } } class ExternalData extends @externalDataElement { + string getDataPath() { externalData(this, result, _, _) } - string getDataPath() { - externalData(this, result, _, _) - } + string getQueryPath() { result = getDataPath().regexpReplaceAll("\\.[^.]*$", ".ql") } - string getQueryPath() { - result = getDataPath().regexpReplaceAll("\\.[^.]*$", ".ql") - } + int getNumFields() { result = 1 + max(int i | externalData(this, _, i, _) | i) } - int getNumFields() { - result = 1 + max(int i | externalData(this, _, i, _) | i) - } + string getField(int index) { externalData(this, _, index, result) } - string getField(int index) { - externalData(this, _, index, result) - } + int getFieldAsInt(int index) { result = getField(index).toInt() } - int getFieldAsInt(int index) { - result = getField(index).toInt() - } + float getFieldAsFloat(int index) { result = getField(index).toFloat() } - float getFieldAsFloat(int index) { - result = getField(index).toFloat() - } + date getFieldAsDate(int index) { result = getField(index).toDate() } - date getFieldAsDate(int index) { - result = getField(index).toDate() - } - - string toString() { - result = getQueryPath() + ": " + buildTupleString(0) - } + string toString() { result = getQueryPath() + ": " + buildTupleString(0) } private string buildTupleString(int start) { - (start = getNumFields() - 1 and result = getField(start)) + start = getNumFields() - 1 and result = getField(start) or - (start < getNumFields() - 1 and result = getField(start) + "," + buildTupleString(start+1)) + start < getNumFields() - 1 and result = getField(start) + "," + buildTupleString(start + 1) } - } /** * External data with a location, and a message, as produced by tools that used to produce QLDs. */ class DefectExternalData extends ExternalData { - DefectExternalData() { - this.getField(0).regexpMatch("\\w+://.*:[0-9]+:[0-9]+:[0-9]+:[0-9]+$") and - this.getNumFields() = 2 - } - - string getURL() { result = getField(0) } + DefectExternalData() { + this.getField(0).regexpMatch("\\w+://.*:[0-9]+:[0-9]+:[0-9]+:[0-9]+$") and + this.getNumFields() = 2 + } - string getMessage() { result = getField(1) } + string getURL() { result = getField(0) } + + string getMessage() { result = getField(1) } } - diff --git a/python/ql/src/external/Thrift.qll b/python/ql/src/external/Thrift.qll index a10d5aab506..658d3f301cd 100644 --- a/python/ql/src/external/Thrift.qll +++ b/python/ql/src/external/Thrift.qll @@ -3,68 +3,42 @@ * This code is under development and may change without warning. */ - import external.ExternalArtifact /** An item in the parse tree of the IDL file */ class ThriftElement extends ExternalData { - string kind; - ThriftElement() { - this.getDataPath() = "thrift-" + kind - } + ThriftElement() { this.getDataPath() = "thrift-" + kind } - string getKind() { - result = kind - } + string getKind() { result = kind } - string getId() { - result = getField(0) - } + string getId() { result = getField(0) } - int getIndex() { - result = getFieldAsInt(1) - } + int getIndex() { result = getFieldAsInt(1) } - ThriftElement getParent() { - result.getId() = this.getField(2) - } + ThriftElement getParent() { result.getId() = this.getField(2) } - string getValue() { - result = this.getField(3) - } + string getValue() { result = this.getField(3) } - ThriftElement getChild(int n) { - result.getIndex() = n and result.getParent() = this - } + ThriftElement getChild(int n) { result.getIndex() = n and result.getParent() = this } - ThriftElement getAChild() { - result = this.getChild(_) - } + ThriftElement getAChild() { result = this.getChild(_) } - override string toString() { - result = this.getKind() - } + override string toString() { result = this.getKind() } - string getPath() { - result = this.getField(4) - } + string getPath() { result = this.getField(4) } - private int line() { - result = this.getFieldAsInt(5) - } + private int line() { result = this.getFieldAsInt(5) } - private int column() { - result = this.getFieldAsInt(6) - } + private int column() { result = this.getFieldAsInt(6) } predicate hasLocationInfo(string fp, int bl, int bc, int el, int ec) { fp = this.getPath() and bl = this.line() and bc = this.column() and el = this.line() and - ec = this.column() + this.getValue().length()-1 + ec = this.column() + this.getValue().length() - 1 or exists(ThriftElement first, ThriftElement last | first = this.getChild(min(int l | exists(this.getChild(l)))) and @@ -74,19 +48,13 @@ class ThriftElement extends ExternalData { ) } - File getFile() { - this.hasLocationInfo(result.getAbsolutePath(), _, _, _, _) - } - + File getFile() { this.hasLocationInfo(result.getAbsolutePath(), _, _, _, _) } } abstract class ThriftNamedElement extends ThriftElement { - abstract ThriftElement getNameElement(); - final string getName() { - result = this.getNameElement().getValue() - } + final string getName() { result = this.getNameElement().getValue() } override string toString() { result = this.getKind() + " " + this.getName() @@ -101,14 +69,10 @@ abstract class ThriftNamedElement extends ThriftElement { this.getNameElement().hasLocationInfo(fp, _, _, el, ec) ) } - } class ThriftType extends ThriftNamedElement { - - ThriftType() { - kind.matches("%type") - } + ThriftType() { kind.matches("%type") } override ThriftElement getNameElement() { result = this.getChild(0) @@ -116,9 +80,7 @@ class ThriftType extends ThriftNamedElement { result = this.getChild(0).(ThriftType).getNameElement() } - override string toString() { - result = "type " + this.getName() - } + override string toString() { result = "type " + this.getName() } predicate references(ThriftStruct struct) { this.getName() = struct.getName() and @@ -127,194 +89,106 @@ class ThriftType extends ThriftNamedElement { struct.hasLocationInfo(path, _, _, _, _) ) } - } /** A thrift typedef */ class ThriftTypeDef extends ThriftNamedElement { + ThriftTypeDef() { kind.matches("typedef") } - ThriftTypeDef() { - kind.matches("typedef") - } - - override ThriftElement getNameElement() { - result = this.getChild(2).getChild(0) - } + override ThriftElement getNameElement() { result = this.getChild(2).getChild(0) } } /** A thrift enum declaration */ class ThriftEnum extends ThriftNamedElement { + ThriftEnum() { kind.matches("enum") } - ThriftEnum() { - kind.matches("enum") - } - - override ThriftElement getNameElement() { - result = this.getChild(0).getChild(0) - } - + override ThriftElement getNameElement() { result = this.getChild(0).getChild(0) } } /** A thrift enum field */ class ThriftEnumField extends ThriftNamedElement { + ThriftEnumField() { kind.matches("enumfield") } - ThriftEnumField() { - kind.matches("enumfield") - } - - override ThriftElement getNameElement() { - result = this.getChild(0).getChild(0) - } - + override ThriftElement getNameElement() { result = this.getChild(0).getChild(0) } } /** A thrift service declaration */ class ThriftService extends ThriftNamedElement { + ThriftService() { kind.matches("service") } - ThriftService() { - kind.matches("service") - } + override ThriftElement getNameElement() { result = this.getChild(0).getChild(0) } - override ThriftElement getNameElement() { - result = this.getChild(0).getChild(0) - } - - ThriftFunction getAFunction() { - result = this.getChild(_) - } + ThriftFunction getAFunction() { result = this.getChild(_) } ThriftFunction getFunction(string name) { result.getName() = name and result = this.getAFunction() } - } /** A thrift function declaration */ class ThriftFunction extends ThriftNamedElement { + ThriftFunction() { kind.matches("function") } - ThriftFunction() { - kind.matches("function") - } + override ThriftElement getNameElement() { result = this.getChild(2).getChild(0) } - override ThriftElement getNameElement() { - result = this.getChild(2).getChild(0) - } + ThriftField getArgument(int n) { result = this.getChild(n + 3) } - ThriftField getArgument(int n) { - result = this.getChild(n+3) - } + ThriftField getAnArgument() { result = this.getArgument(_) } - ThriftField getAnArgument() { - result = this.getArgument(_) - } + private ThriftThrows getAllThrows() { result = this.getChild(_) } - private ThriftThrows getAllThrows() { - result = this.getChild(_) - } + ThriftField getAThrows() { result = this.getAllThrows().getAChild() } - ThriftField getAThrows() { - result = this.getAllThrows().getAChild() - } - - ThriftType getReturnType() { - result = this.getChild(1).getChild(0) - } + ThriftType getReturnType() { result = this.getChild(1).getChild(0) } override predicate hasLocationInfo(string fp, int bl, int bc, int el, int ec) { this.getChild(1).hasLocationInfo(fp, bl, bc, _, _) and this.getChild(2).hasLocationInfo(fp, _, _, el, ec) } - ThriftService getService() { - result.getAFunction() = this - } - - string getQualifiedName() { - result = this.getService().getName() + "." + this.getName() - } + ThriftService getService() { result.getAFunction() = this } + string getQualifiedName() { result = this.getService().getName() + "." + this.getName() } } class ThriftField extends ThriftNamedElement { + ThriftField() { kind.matches("field") } - ThriftField() { - kind.matches("field") - } - - override ThriftElement getNameElement() { - result = this.getChild(4) - } - - ThriftType getType() { - result = this.getChild(2) - } + override ThriftElement getNameElement() { result = this.getChild(4) } + ThriftType getType() { result = this.getChild(2) } } -class ThriftStruct extends ThriftNamedElement { +class ThriftStruct extends ThriftNamedElement { + ThriftStruct() { kind.matches("struct") } - ThriftStruct() { - kind.matches("struct") - } + override ThriftElement getNameElement() { result = this.getChild(0).getChild(0) } - override ThriftElement getNameElement() { - result = this.getChild(0).getChild(0) - } - - ThriftField getMember(int n) { - result = this.getChild(n+1) - } - - ThriftField getAMember() { - result = this.getMember(_) - } + ThriftField getMember(int n) { result = this.getChild(n + 1) } + ThriftField getAMember() { result = this.getMember(_) } } - class ThriftException extends ThriftNamedElement { + ThriftException() { kind.matches("exception") } - ThriftException() { - kind.matches("exception") - } + override ThriftElement getNameElement() { result = this.getChild(0).getChild(0) } - override ThriftElement getNameElement() { - result = this.getChild(0).getChild(0) - } - - ThriftField getMember(int n) { - result = this.getChild(n+1) - } - - ThriftField getAMember() { - result = this.getMember(_) - } + ThriftField getMember(int n) { result = this.getChild(n + 1) } + ThriftField getAMember() { result = this.getMember(_) } } - class ThriftThrows extends ThriftElement { + ThriftThrows() { kind.matches("throws") } - ThriftThrows() { - kind.matches("throws") - } - - ThriftField getAThrows() { - result = this.getChild(_) - } - + ThriftField getAThrows() { result = this.getChild(_) } } /** A parse tree element that holds a primitive value */ class ThriftValue extends ThriftElement { + ThriftValue() { exists(this.getValue()) } - ThriftValue() { - exists(this.getValue()) - } - - override string toString() { - result = this.getKind() + " " + this.getValue() - } - + override string toString() { result = this.getKind() + " " + this.getValue() } } diff --git a/python/ql/src/external/VCS.qll b/python/ql/src/external/VCS.qll index 6b665dde510..2d03f7d882d 100644 --- a/python/ql/src/external/VCS.qll +++ b/python/ql/src/external/VCS.qll @@ -1,92 +1,76 @@ import python class Commit extends @svnentry { + Commit() { + svnaffectedfiles(this, _, _) and + exists(date svnDate, date snapshotDate | + svnentries(this, _, _, svnDate, _) and + snapshotDate(snapshotDate) and + svnDate <= snapshotDate + ) + } - Commit() { - svnaffectedfiles(this, _, _) and - exists(date svnDate, date snapshotDate | - svnentries(this, _, _, svnDate, _) and - snapshotDate(snapshotDate) and - svnDate <= snapshotDate - ) - } + string toString() { result = this.getRevisionName() } - string toString() { result = this.getRevisionName() } + string getRevisionName() { svnentries(this, result, _, _, _) } - string getRevisionName() { svnentries(this, result, _, _, _) } - - string getAuthor() { svnentries(this, _, result, _, _) } - - date getDate() { svnentries(this, _, _, result, _) } - - int getChangeSize() { svnentries(this, _, _, _, result) } - - string getMessage() { svnentrymsg(this, result) } - - string getAnAffectedFilePath(string action) { - exists(File rawFile | svnaffectedfiles(this, rawFile, action) | - result = rawFile.getName() - ) - } - - string getAnAffectedFilePath() { result = getAnAffectedFilePath(_) } - - File getAnAffectedFile(string action) { - svnaffectedfiles(this,result,action) - } + string getAuthor() { svnentries(this, _, result, _, _) } - File getAnAffectedFile() { exists(string action | result = this.getAnAffectedFile(action)) } + date getDate() { svnentries(this, _, _, result, _) } - predicate isRecent() { recentCommit(this) } - - int daysToNow() { - exists(date now | snapshotDate(now) | - result = getDate().daysTo(now) and result >= 0 - ) - } - - int getRecentAdditionsForFile(File f) { - svnchurn(this, f, result, _) - } - - int getRecentDeletionsForFile(File f) { - svnchurn(this, f, _, result) - } + int getChangeSize() { svnentries(this, _, _, _, result) } - int getRecentChurnForFile(File f) { - result = getRecentAdditionsForFile(f) + getRecentDeletionsForFile(f) - } + string getMessage() { svnentrymsg(this, result) } + string getAnAffectedFilePath(string action) { + exists(File rawFile | svnaffectedfiles(this, rawFile, action) | result = rawFile.getName()) + } + + string getAnAffectedFilePath() { result = getAnAffectedFilePath(_) } + + File getAnAffectedFile(string action) { svnaffectedfiles(this, result, action) } + + File getAnAffectedFile() { exists(string action | result = this.getAnAffectedFile(action)) } + + predicate isRecent() { recentCommit(this) } + + int daysToNow() { + exists(date now | snapshotDate(now) | result = getDate().daysTo(now) and result >= 0) + } + + int getRecentAdditionsForFile(File f) { svnchurn(this, f, result, _) } + + int getRecentDeletionsForFile(File f) { svnchurn(this, f, _, result) } + + int getRecentChurnForFile(File f) { + result = getRecentAdditionsForFile(f) + getRecentDeletionsForFile(f) + } } class Author extends string { - Author() { exists(Commit e | this = e.getAuthor()) } - - Commit getACommit() { result.getAuthor() = this } - - File getAnEditedFile() { result = this.getACommit().getAnAffectedFile() } - + Author() { exists(Commit e | this = e.getAuthor()) } + + Commit getACommit() { result.getAuthor() = this } + + File getAnEditedFile() { result = this.getACommit().getAnAffectedFile() } } predicate recentCommit(Commit e) { - exists(date snapshotDate, date commitDate, int days | - snapshotDate(snapshotDate) and - e.getDate() = commitDate and - days = commitDate.daysTo(snapshotDate) and - days >= 0 and days <= 60 - ) + exists(date snapshotDate, date commitDate, int days | + snapshotDate(snapshotDate) and + e.getDate() = commitDate and + days = commitDate.daysTo(snapshotDate) and + days >= 0 and + days <= 60 + ) } date firstChange(File f) { - result = min(Commit e, date toMin | (f = e.getAnAffectedFile()) and (toMin = e.getDate()) | toMin) + result = min(Commit e, date toMin | f = e.getAnAffectedFile() and toMin = e.getDate() | toMin) } predicate firstCommit(Commit e) { - not exists(File f | f = e.getAnAffectedFile() | - firstChange(f) < e.getDate() - ) + not exists(File f | f = e.getAnAffectedFile() | firstChange(f) < e.getDate()) } -predicate artificialChange(Commit e) { - firstCommit(e) or e.getChangeSize() >= 50000 -} \ No newline at end of file +predicate artificialChange(Commit e) { firstCommit(e) or e.getChangeSize() >= 50000 } diff --git a/python/ql/src/python.qll b/python/ql/src/python.qll index d1c096ff4ac..6cf0802d993 100644 --- a/python/ql/src/python.qll +++ b/python/ql/src/python.qll @@ -34,5 +34,4 @@ import semmle.python.pointsto.Base import semmle.python.pointsto.Context import semmle.python.pointsto.CallGraph import semmle.python.objects.ObjectAPI - import site diff --git a/python/ql/src/semmle/crypto/Crypto.qll b/python/ql/src/semmle/crypto/Crypto.qll index 12e81a393ce..4eaa3eb8dbd 100644 --- a/python/ql/src/semmle/crypto/Crypto.qll +++ b/python/ql/src/semmle/crypto/Crypto.qll @@ -2,10 +2,10 @@ * Provides classes for modeling cryptographic libraries. */ -/* The following information is copied from `/semmlecode-javascript-queries/semmle/javascript/frameworks/CryptoLibraries.qll` +/* + * The following information is copied from `/semmlecode-javascript-queries/semmle/javascript/frameworks/CryptoLibraries.qll` * which should be considered the definitive version (as of Feb 2018) */ - /** * Names of cryptographic algorithms, separated into strong and weak variants. @@ -13,15 +13,17 @@ * The names are normalized: upper-case, no spaces, dashes or underscores. * * The names are inspired by the names used in real world crypto libraries. - * */ private module AlgorithmNames { predicate isStrongHashingAlgorithm(string name) { name = "DSA" or name = "ED25519" or - name = "ES256" or name = "ECDSA256" or - name = "ES384" or name = "ECDSA384" or - name = "ES512" or name = "ECDSA512" or + name = "ES256" or + name = "ECDSA256" or + name = "ES384" or + name = "ECDSA384" or + name = "ES512" or + name = "ECDSA512" or name = "SHA2" or name = "SHA224" or name = "SHA256" or @@ -54,15 +56,21 @@ private module AlgorithmNames { name = "RSA" or name = "RABBIT" or name = "BLOWFISH" - } predicate isWeakEncryptionAlgorithm(string name) { name = "DES" or - name = "3DES" or name = "TRIPLEDES" or name = "TDEA" or name = "TRIPLEDEA" or - name = "ARC2" or name = "RC2" or - name = "ARC4" or name = "RC4" or name = "ARCFOUR" or - name = "ARC5" or name = "RC5" + name = "3DES" or + name = "TRIPLEDES" or + name = "TDEA" or + name = "TRIPLEDEA" or + name = "ARC2" or + name = "RC2" or + name = "ARC4" or + name = "RC4" or + name = "ARCFOUR" or + name = "ARC5" or + name = "RC5" } predicate isStrongPasswordHashingAlgorithm(string name) { @@ -72,51 +80,45 @@ private module AlgorithmNames { name = "SCRYPT" } - predicate isWeakPasswordHashingAlgorithm(string name) { - none() - } + predicate isWeakPasswordHashingAlgorithm(string name) { none() } /** * Normalizes `name`: upper-case, no spaces, dashes or underscores. * * All names of this module are in this normalized form. */ - bindingset[name] string normalizeName(string name) { - result = name.toUpperCase().regexpReplaceAll("[-_ ]", "") - } - + bindingset[name] + string normalizeName(string name) { result = name.toUpperCase().regexpReplaceAll("[-_ ]", "") } } -private import AlgorithmNames +private import AlgorithmNames /** * A cryptographic algorithm. */ private newtype TCryptographicAlgorithm = -MkHashingAlgorithm(string name, boolean isWeak) { - (isStrongHashingAlgorithm(name) and isWeak = false) or - (isWeakHashingAlgorithm(name) and isWeak = true) -} -or -MkEncryptionAlgorithm(string name, boolean isWeak) { - (isStrongEncryptionAlgorithm(name) and isWeak = false) or - (isWeakEncryptionAlgorithm(name) and isWeak = true) -} -or -MkPasswordHashingAlgorithm(string name, boolean isWeak) { - (isStrongPasswordHashingAlgorithm(name) and isWeak = false) or - (isWeakPasswordHashingAlgorithm(name) and isWeak = true) -} + MkHashingAlgorithm(string name, boolean isWeak) { + isStrongHashingAlgorithm(name) and isWeak = false + or + isWeakHashingAlgorithm(name) and isWeak = true + } or + MkEncryptionAlgorithm(string name, boolean isWeak) { + isStrongEncryptionAlgorithm(name) and isWeak = false + or + isWeakEncryptionAlgorithm(name) and isWeak = true + } or + MkPasswordHashingAlgorithm(string name, boolean isWeak) { + isStrongPasswordHashingAlgorithm(name) and isWeak = false + or + isWeakPasswordHashingAlgorithm(name) and isWeak = true + } /** * A cryptographic algorithm. */ abstract class CryptographicAlgorithm extends TCryptographicAlgorithm { - /** Gets a textual representation of this element. */ - string toString() { - result = getName() - } + string toString() { result = getName() } /** * Gets the name of the algorithm. @@ -127,76 +129,46 @@ abstract class CryptographicAlgorithm extends TCryptographicAlgorithm { * Holds if this algorithm is weak. */ abstract predicate isWeak(); - } /** * A hashing algorithm such as `MD5` or `SHA512`. */ class HashingAlgorithm extends MkHashingAlgorithm, CryptographicAlgorithm { - string name; - boolean isWeak; - HashingAlgorithm() { - this = MkHashingAlgorithm(name, isWeak) - } + HashingAlgorithm() { this = MkHashingAlgorithm(name, isWeak) } - override string getName() { - result = name - } - - override predicate isWeak() { - isWeak = true - } + override string getName() { result = name } + override predicate isWeak() { isWeak = true } } /** * An encryption algorithm such as `DES` or `AES512`. */ class EncryptionAlgorithm extends MkEncryptionAlgorithm, CryptographicAlgorithm { - string name; - boolean isWeak; - EncryptionAlgorithm() { - this = MkEncryptionAlgorithm(name, isWeak) - } + EncryptionAlgorithm() { this = MkEncryptionAlgorithm(name, isWeak) } - override string getName() { - result = name - } - - override predicate isWeak() { - isWeak = true - } + override string getName() { result = name } + override predicate isWeak() { isWeak = true } } /** * A password hashing algorithm such as `PBKDF2` or `SCRYPT`. */ -class PasswordHashingAlgorithm extends MkPasswordHashingAlgorithm, CryptographicAlgorithm { - +class PasswordHashingAlgorithm extends MkPasswordHashingAlgorithm, CryptographicAlgorithm { string name; - boolean isWeak; - PasswordHashingAlgorithm() { - this = MkPasswordHashingAlgorithm(name, isWeak) - } + PasswordHashingAlgorithm() { this = MkPasswordHashingAlgorithm(name, isWeak) } - override string getName() { - result = name - } + override string getName() { result = name } - override predicate isWeak() { - isWeak = true - } + override predicate isWeak() { isWeak = true } } - - - diff --git a/python/ql/src/semmle/files/FileSystem.qll b/python/ql/src/semmle/files/FileSystem.qll index 4ec67c7c2e6..4e28b33d96d 100644 --- a/python/ql/src/semmle/files/FileSystem.qll +++ b/python/ql/src/semmle/files/FileSystem.qll @@ -1,2 +1,3 @@ /** Provides classes for working with files and folders. */ + import semmle.python.Files