Merge pull request #6928 from RasmusWL/diagnostic-as-warning

Python: Improve SARIF severity level reporting of extractor diagnostics

python/change-notes/2021-10-20-extraction-errors-as-warnings.md

@@ -0,0 +1,2 @@
+codescanning
+* Problems with extraction that in most cases won't completely break the analysis are now reported as warnings rather than errors.

python/ql/src/Diagnostics/ExtractionErrors.ql

@@ -1,23 +0,0 @@
-/**
- * @name Python extraction errors
- * @description List all extraction errors for Python files in the source code directory.
- * @kind diagnostic
- * @id py/diagnostics/extraction-errors
- */
-
-import python
-
-/**
- * Gets the SARIF severity for errors.
- *
- * See point 3.27.10 in https://docs.oasis-open.org/sarif/sarif/v2.0/sarif-v2.0.html for
- * what error means.
- */
-int getErrorSeverity() { result = 2 }
-
-from SyntaxError error, File file
-where
-  file = error.getFile() and
-  exists(file.getRelativePath())
-select error, "Extraction failed in " + file + " with error " + error.getMessage(),
-  getErrorSeverity()

python/ql/src/Diagnostics/ExtractionWarnings.ql

@@ -0,0 +1,36 @@
+/**
+ * @name Python extraction warnings
+ * @description List all extraction warnings for Python files in the source code directory.
+ * @kind diagnostic
+ * @id py/diagnostics/extraction-warnings
+ */
+
+import python
+
+/**
+ * Gets the SARIF severity for warnings.
+ *
+ * See https://docs.oasis-open.org/sarif/sarif/v2.1.0/csprd01/sarif-v2.1.0-csprd01.html#_Toc10541338
+ */
+int getWarningSeverity() { result = 1 }
+
+// The spec
+// https://docs.oasis-open.org/sarif/sarif/v2.1.0/csprd01/sarif-v2.1.0-csprd01.html#_Toc10541338
+// defines error and warning as:
+//
+// "error": A serious problem was found. The condition encountered by the tool resulted
+// in the analysis being halted or caused the results to be incorrect or incomplete.
+//
+// "warning": A problem that is not considered serious was found. The condition
+// encountered by the tool is such that it is uncertain whether a problem occurred, or
+// is such that the analysis might be incomplete but the results that were generated are
+// probably valid.
+//
+// So SyntaxErrors are reported at the warning level, since analysis might be incomplete
+// but the results that were generated are probably valid.
+from SyntaxError error, File file
+where
+  file = error.getFile() and
+  exists(file.getRelativePath())
+select error, "Extraction failed in " + file + " with error " + error.getMessage(),
+  getWarningSeverity()

python/ql/test/query-tests/Diagnostics/ExtractionErrors.qlref

@@ -1 +0,0 @@
-Diagnostics/ExtractionErrors.ql

python/ql/test/query-tests/Diagnostics/ExtractionWarnings.expected

@@ -1,2 +1,2 @@
-| bad_encoding.py:2:11:2:11 | Encoding Error | Extraction failed in bad_encoding.py with error 'utf-8' codec can't decode byte 0x9d in position 87: invalid start byte | 2 |
-| syntax_error.py:1:31:1:31 | Syntax Error | Extraction failed in syntax_error.py with error Syntax Error | 2 |
+| bad_encoding.py:2:11:2:11 | Encoding Error | Extraction failed in bad_encoding.py with error 'utf-8' codec can't decode byte 0x9d in position 87: invalid start byte | 1 |
+| syntax_error.py:1:31:1:31 | Syntax Error | Extraction failed in syntax_error.py with error Syntax Error | 1 |

python/ql/test/query-tests/Diagnostics/ExtractionWarnings.qlref

@@ -0,0 +1 @@
+Diagnostics/ExtractionWarnings.ql