Merge pull request #5755 from RasmusWL/non-alert-data-part1

Approved by tausbn

python/ql/src/Diagnostics/ExtractionErrors.ql

@@ -0,0 +1,23 @@
+/**
+ * @name Python extraction errors
+ * @description List all extraction errors for Python files in the source code directory.
+ * @kind diagnostic
+ * @id py/diagnostics/extraction-errors
+ */
+
+import python
+
+/**
+ * Gets the SARIF severity for errors.
+ *
+ * See point 3.27.10 in https://docs.oasis-open.org/sarif/sarif/v2.0/sarif-v2.0.html for
+ * what error means.
+ */
+int getErrorSeverity() { result = 2 }
+
+from SyntaxError error, File file
+where
+  file = error.getFile() and
+  exists(file.getRelativePath())
+select error, "Extraction failed in " + file + " with error " + error.getMessage(),
+  getErrorSeverity()

python/ql/src/Diagnostics/SuccessfullyExtractedFiles.ql

@@ -0,0 +1,15 @@
+/**
+ * @name Successfully extracted Python files
+ * @description Lists all Python files in the source code directory that were extracted
+ *   without encountering an error.
+ * @kind diagnostic
+ * @id py/diagnostics/successfully-extracted-files
+ */
+
+import python
+
+from File file
+where
+  not exists(SyntaxError e | e.getFile() = file) and
+  exists(file.getRelativePath())
+select file, ""

python/ql/src/Summary/LinesOfCode.ql

@@ -0,0 +1,13 @@
+/**
+ * @name Total lines of Python code in the database
+ * @description The total number of lines of Python code across all files, including
+ *   external libraries and auto-generated files. This is a useful metric of the size of a
+ *   database. This query counts the lines of code, excluding whitespace or comments.
+ * @kind metric
+ * @tags summary
+ * @id py/summary/lines-of-code
+ */
+
+import python
+
+select sum(Module m | | m.getMetrics().getNumberOfLinesOfCode())

python/ql/src/Summary/LinesOfUserCode.ql

@@ -0,0 +1,21 @@
+/**
+ * @name Total lines of user written Python code in the database
+ * @description The total number of lines of Python code from the source code directory,
+ *   excluding auto-generated files. This query counts the lines of code, excluding
+ *   whitespace or comments. Note: If external libraries are included in the codebase
+ *   either in a checked-in virtual environment or as vendored code, that will currently
+ *   be counted as user written code.
+ * @kind metric
+ * @tags summary
+ * @id py/summary/lines-of-user-code
+ */
+
+import python
+import semmle.python.filters.GeneratedCode
+
+select sum(Module m |
+    exists(m.getFile().getRelativePath()) and
+    not m.getFile() instanceof GeneratedFile
+  |
+    m.getMetrics().getNumberOfLinesOfCode()
+  )

python/ql/test/query-tests/Diagnostics/ExtractionErrors.expected

@@ -0,0 +1,2 @@
+| bad_encoding.py:2:11:2:11 | Encoding Error | Extraction failed in bad_encoding.py with error 'utf-8' codec can't decode byte 0x9d in position 87: invalid start byte | 2 |
+| syntax_error.py:1:31:1:31 | Syntax Error | Extraction failed in syntax_error.py with error Syntax Error | 2 |

python/ql/test/query-tests/Diagnostics/ExtractionErrors.qlref

@@ -0,0 +1 @@
+Diagnostics/ExtractionErrors.ql

python/ql/test/query-tests/Diagnostics/SuccessfullyExtractedFiles.expected

@@ -0,0 +1 @@
+| good_file.py:0:0:0:0 | good_file.py |  |

python/ql/test/query-tests/Diagnostics/SuccessfullyExtractedFiles.qlref

@@ -0,0 +1 @@
+Diagnostics/SuccessfullyExtractedFiles.ql

python/ql/test/query-tests/Diagnostics/bad_encoding.py

@@ -0,0 +1,2 @@
+# Note: This file has been encoded in Windows 1252 to provoke encoding error
+print("wat<EFBFBD>")

python/ql/test/query-tests/Diagnostics/good_file.py

@@ -0,0 +1 @@
+print("hello world")

python/ql/test/query-tests/Diagnostics/options

@@ -0,0 +1 @@
+semmle-extractor-options: --max-import-depth=1 --lang=3

python/ql/test/query-tests/Diagnostics/syntax_error.py

@@ -0,0 +1 @@
+print("no closing parenthesis"

python/ql/test/query-tests/Summary/LinesOfCode.expected

@@ -0,0 +1 @@
+| 38 |

python/ql/test/query-tests/Summary/LinesOfCode.qlref

@@ -0,0 +1 @@
+Summary/LinesOfCode.ql

python/ql/test/query-tests/Summary/LinesOfUserCode.expected

@@ -0,0 +1 @@
+| 11 |

python/ql/test/query-tests/Summary/LinesOfUserCode.qlref

@@ -0,0 +1 @@
+Summary/LinesOfUserCode.ql

python/ql/test/query-tests/Summary/also_python_code

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+
+# although this is actually Python code, it is not included by the extractor by default.
+
+print("this is also code")
+
+print("but just dummy code")

python/ql/test/query-tests/Summary/my_file.py

@@ -0,0 +1,26 @@
+"""
+module level docstring
+
+is not included
+"""
+# this line is not code
+
+# `tty` was chosen for stability over python versions (so we don't get diffrent results
+# on different computers, that has different versions of Python).
+#
+# According to https://github.com/python/cpython/tree/master/Lib (at 2021-04-23) `tty`
+# was last changed in 2001, so chances of this being changed in the future are slim.
+import tty
+
+s = """
+all these lines are code
+"""
+
+print(s)
+
+def func():
+    """
+    this string is a doc-string. Although the module-level docstring is not considered
+    code, this one apparently is ¯\_(ツ)_/¯
+    """
+    pass

python/ql/test/query-tests/Summary/not_python

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# Although this is valid python code, it should not be counted as such.
+
+print("foo")