Merge pull request #531 from sauyon/non-alert-queries

Non-alert queries

Makefile

@@ -30,12 +30,12 @@ clean:
 DATAFLOW_BRANCH=main
 
 autoformat:
-	find ql -name "*.ql" -or -name "*.qll" | xargs codeql query format -qq -i
-	git ls-files | grep '\.go$$' | grep -v ^vendor/ | xargs grep -L "//\s*autoformat-ignore" | xargs gofmt -w
+	find ql -iregex '.*\.qll?' -print0 | xargs -0 codeql query format -qq -i
+	find . -path '**/vendor' -prune -or -type f -iname '*.go' ! -empty -print0 | xargs -0 grep -L "//\s*autoformat-ignore" | xargs gofmt -w
 
 check-formatting:
-	find ql -name "*.ql" -or -name "*.qll" | xargs codeql query format --check-only
-	test -z "$$(git ls-files | grep '\.go$$' | grep -v ^vendor/ | xargs grep -L "//\s*autoformat-ignore" | xargs gofmt -l)"
+	find ql -iregex '.*\.qll?' -print0 | xargs -0 codeql query format --check-only
+	test -z "$$(find . -path '**/vendor' -prune -or -type f -iname '*.go' ! -empty -print0 | xargs -0 grep -L "//\s*autoformat-ignore" | xargs gofmt -l)"
 
 ifeq ($(QHELP_OUT_DIR),)
 # If not otherwise specified, compile qhelp to markdown in place

extractor/extractor.go

@@ -460,11 +460,17 @@ func (extraction *Extraction) extractError(tw *trap.Writer, err packages.Error,
 
 	if pos == "" {
 		// extract a dummy file
-		file, e = filepath.Abs(filepath.Join(".", "-"))
+		wd, e := os.Getwd()
 		if e != nil {
-			file = filepath.Join(".", "-")
-			log.Printf("Warning: failed to get absolute path for for %s", file)
+			wd = "."
+			log.Printf("Warning: failed to get working directory")
 		}
+		ewd, e := filepath.EvalSymlinks(wd)
+		if e != nil {
+			ewd = wd
+			log.Printf("Warning: failed to evaluate symlinks for %s", wd)
+		}
+		file = filepath.Join(ewd, "-")
 	} else {
 		var rawfile string
 		if parts := threePartPos.FindStringSubmatch(pos); parts != nil {
@@ -1623,11 +1629,19 @@ func extractNumLines(tw *trap.Writer, fileName string, ast *ast.File) {
 		pos, tok, lit := s.Scan()
 		if tok == token.EOF {
 			break
-		} else if tok != token.ILLEGAL {
+		} else if tok != token.ILLEGAL && !(tok == token.SEMICOLON && lit == "\n") {
+			// specifically exclude newlines that are treated as semicolons
 			tkStartLine := f.Position(pos).Line
 			tkEndLine := tkStartLine + strings.Count(lit, "\n")
 			if tkEndLine > lastCodeLine {
-				linesOfCode += tkEndLine - tkStartLine + 1
+				if tkStartLine <= lastCodeLine {
+					// if the start line is the same as the last code line we've seen we don't want to double
+					// count it
+					// note tkStartLine < lastCodeLine should not be possible
+					linesOfCode += tkEndLine - lastCodeLine
+				} else {
+					linesOfCode += tkEndLine - tkStartLine + 1
+				}
 				lastCodeLine = tkEndLine
 			}
 		}

ql/src/Diagnostics/DiagnosticsReporting.qll

@@ -0,0 +1,54 @@
+import go
+
+/** Gets the SARIF severity level that indicates an error. */
+private int getErrorSeverity() { result = 2 }
+
+private class Diagnostic extends @diagnostic {
+  /**
+   * Gets the kind of error. This can be:
+   * * `@unknownerror`: an unknown error
+   * * `@listerror`: an error from the frontend
+   * * `@parseerror`: a parse error
+   * * `@typeerror`: a type error
+   */
+  string getKind() { diagnostics(this, _, result, _, _, _) }
+
+  /** Gets the error message for this error. */
+  string getMessage() { diagnostics(this, _, _, result, _, _) }
+
+  /** Gets the file that this error is associated with, if any. */
+  File getFile() { this.hasLocationInfo(result.getAbsolutePath(), _, _, _, _) }
+
+  /**
+   * Holds if this element is at the specified location.
+   * The location spans column `startcolumn` of line `startline` to
+   * column `endcolumn` of line `endline` in file `filepath`.
+   * For more information, see
+   * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html).
+   */
+  predicate hasLocationInfo(string path, int sl, int sc, int el, int ec) {
+    exists(Location l | diagnostics(this, _, _, _, _, l) | l.hasLocationInfo(path, sl, sc, el, ec))
+  }
+
+  string toString() { result = this.getMessage() }
+}
+
+/**
+ * Holds if an extraction error or warning occurred that should be reported to end users,
+ * with the error message `msg` and SARIF severity `sev`.
+ */
+predicate reportableDiagnostics(Diagnostic d, string msg, int sev) {
+  // Go does not have warnings, so all errors have error severity
+  sev = getErrorSeverity() and
+  (
+    // Only report errors for files that would have been extracted
+    exists(File f | f = d.getFile() |
+      exists(f.getAChild()) and
+      msg =
+        "Extraction failed in " + d.getFile().getRelativePath() + " with error " + d.getMessage()
+    )
+    or
+    not exists(d.getFile()) and
+    msg = "Extraction failed with error " + d.getMessage()
+  )
+}

ql/src/Diagnostics/ExtractionErrors.ql

@@ -0,0 +1,13 @@
+/**
+ * @id go/diagnostics/extraction-errors
+ * @name Extraction errors
+ * @description List all extraction errors for files in the source code directory.
+ * @kind diagnostic
+ */
+
+import go
+import DiagnosticsReporting
+
+from string msg, int sev
+where reportableDiagnostics(_, msg, sev)
+select msg, sev

ql/src/Diagnostics/SuccessfullyExtractedFiles.ql

@@ -0,0 +1,12 @@
+/**
+ * @id go/summary/successfully-extracted-files
+ * @name Successfully analyzed files
+ * @description List all files that were successfully extracted.
+ * @kind diagnostic
+ */
+
+import go
+
+from File f
+where not exists(Error e | e.getFile() = f)
+select f.getRelativePath()

ql/src/Summary/LinesOfCode.ql

@@ -0,0 +1,11 @@
+/**
+ * @id go/summary/lines-of-code
+ * @name Total lines of Go code in the database
+ * @description The total number of lines of Go code across all extracted files, including auto-generated files. This is a useful metric of the size of a database. For all files that were seen during the build, this query counts the lines of code, excluding whitespace or comments.
+ * @kind metric
+ * @tags summary
+ */
+
+import go
+
+select sum(GoFile f | | f.getNumberOfLinesOfCode())

ql/src/filters/ClassifyFiles.ql

@@ -8,17 +8,6 @@
 
 import go
 
-string generatorCommentRegex() {
-  result = "Generated By\\b.*\\bDo not edit" or
-  result = "This (file|class|interface|art[ei]fact) (was|is|(has been)) (?:auto[ -]?)?gener(e?)ated" or
-  result = "Any modifications to this file will be lost" or
-  result =
-    "This (file|class|interface|art[ei]fact) (was|is) (?:mechanically|automatically) generated" or
-  result = "The following code was (?:auto[ -]?)?generated (?:by|from)" or
-  result = "Autogenerated by Thrift" or
-  result = "(Code g|G)enerated from .* by ANTLR"
-}
-
 predicate classify(File f, string category) {
   // tests
   f instanceof TestFile and
@@ -29,13 +18,7 @@ predicate classify(File f, string category) {
   category = "library"
   or
   // generated code
-  exists(Comment c | c.getFile() = f |
-    c.getText().regexpMatch("(?i).*\\b(" + concat(generatorCommentRegex(), "|") + ")\\b.*")
-    or
-    // regular expression recommended for Go code generators
-    // (https://golang.org/pkg/cmd/go/internal/generate/)
-    c.getText().regexpMatch("^\\s*Code generated .* DO NOT EDIT\\.\\s*$")
-  ) and
+  f instanceof GeneratedFile and
   category = "generated"
 }
 

ql/src/go.qll

@@ -21,7 +21,6 @@ import semmle.go.StringOps
 import semmle.go.Types
 import semmle.go.Util
 import semmle.go.VariableWithFields
-import semmle.go.concepts.HTTP
 import semmle.go.controlflow.BasicBlocks
 import semmle.go.controlflow.ControlFlowGraph
 import semmle.go.controlflow.IR

ql/src/semmle/go/Concepts.qll

@@ -6,6 +6,8 @@
 
 import go
 import semmle.go.dataflow.FunctionInputsAndOutputs
+import semmle.go.concepts.HTTP
+import semmle.go.concepts.GeneratedFile
 
 /**
  * A data-flow node that executes an operating system command,

ql/src/semmle/go/Files.qll

@@ -264,3 +264,13 @@ class File extends ExtractedOrExternalFile {
     exists(this.getAChild())
   }
 }
+
+/** A Go file. */
+class GoFile extends File {
+  GoFile() { this.getExtension() = "go" }
+}
+
+/** An HTML file. */
+class HtmlFile extends File {
+  HtmlFile() { this.getExtension().regexpMatch("x?html?") }
+}

ql/src/semmle/go/HTML.qll

@@ -3,13 +3,6 @@
 import go
 
 module HTML {
-  /**
-   * An HTML file.
-   */
-  class HtmlFile extends File {
-    HtmlFile() { this.getExtension().regexpMatch("x?html?") }
-  }
-
   /**
    * An HTML element.
    *

ql/src/semmle/go/concepts/GeneratedFile.qll

@@ -0,0 +1,50 @@
+/** Provides a class for generated files. */
+
+import go
+
+/** Provides a class for generated files. */
+module GeneratedFile {
+  /**
+   * A file that has been generated.
+   *
+   * Extend this class to model new APIs. If you want to refine existing API models,
+   * extend `GeneratedFile` instead.
+   */
+  abstract class Range extends File { }
+
+  private string generatorCommentRegex() {
+    result = "Generated By\\b.*\\bDo not edit" or
+    result =
+      "This (file|class|interface|art[ei]fact) (was|is|(has been)) (?:auto[ -]?)?gener(e?)ated" or
+    result = "Any modifications to this file will be lost" or
+    result =
+      "This (file|class|interface|art[ei]fact) (was|is) (?:mechanically|automatically) generated" or
+    result = "The following code was (?:auto[ -]?)?generated (?:by|from)" or
+    result = "Autogenerated by Thrift" or
+    result = "(Code g|G)enerated from .* by ANTLR"
+  }
+
+  private class CommentHeuristicGeneratedFile extends Range {
+    CommentHeuristicGeneratedFile() {
+      exists(Comment c | c.getFile() = this |
+        c.getText().regexpMatch("(?i).*\\b(" + concat(generatorCommentRegex(), "|") + ")\\b.*")
+        or
+        // regular expression recommended for Go code generators
+        // (https://golang.org/pkg/cmd/go/internal/generate/)
+        c.getText().regexpMatch("^\\s*Code generated .* DO NOT EDIT\\.\\s*$")
+      )
+    }
+  }
+}
+
+/**
+ * A file that has been generated.
+ *
+ * Extend this class to refine existing API models. If you want to model new APIs,
+ * extend `GeneratedFile::Range` instead.
+ */
+class GeneratedFile extends File {
+  GeneratedFile::Range self;
+
+  GeneratedFile() { this = self }
+}

ql/test/query-tests/Diagnostics/CONSISTENCY/UnexpectedFrontendErrors.expected

@@ -0,0 +1,3 @@
+| -:0:0:0:0 | malformed import path "github.com/github/codeql-go/ql/test/query-tests/Diagnostics/invalid{": invalid char '{' |
+| bad.go:3:1:3:1 | expected 'package', found avvu |
+| type.go:11:9:11:9 | cannot use v (variable of type V) as T value in argument to takesT |

ql/test/query-tests/Diagnostics/ExtractionErrors.expected

@@ -0,0 +1,3 @@
+| Extraction failed in query-tests/Diagnostics/type.go with error cannot use v (variable of type V) as T value in argument to takesT | 2 |
+| Extraction failed with error expected 'package', found avvu | 2 |
+| Extraction failed with error malformed import path "github.com/github/codeql-go/ql/test/query-tests/Diagnostics/invalid{": invalid char '{' | 2 |

ql/test/query-tests/Diagnostics/ExtractionErrors.qlref

@@ -0,0 +1 @@
+Diagnostics/ExtractionErrors.ql

ql/test/query-tests/Diagnostics/SuccessfullyExtractedFiles.expected

@@ -0,0 +1 @@
+| query-tests/Diagnostics/util.go |

ql/test/query-tests/Diagnostics/SuccessfullyExtractedFiles.qlref

@@ -0,0 +1 @@
+Diagnostics/SuccessfullyExtractedFiles.ql

ql/test/query-tests/Diagnostics/bad.go

@@ -0,0 +1,5 @@
+// autoformat-ignore
+
+avvu
+
+wnvwun

ql/test/query-tests/Diagnostics/badimport.go

@@ -0,0 +1,7 @@
+// autoformat-ignore
+
+package main
+
+import (
+	"github.com/pkg{}"
+)

ql/test/query-tests/Diagnostics/invalid{/invalid.go

@@ -0,0 +1 @@
+package invalid

ql/test/query-tests/Diagnostics/type.go

@@ -0,0 +1,12 @@
+package main
+
+type T int
+
+type V int
+
+func takesT(t T) {}
+
+func passesV() {
+	var v V
+	takesT(v)
+}

ql/test/query-tests/Diagnostics/util.go

@@ -0,0 +1,5 @@
+package main
+
+func use(_ ...interface{}) {
+
+}

ql/test/query-tests/Summary/CONSISTENCY/UnexpectedFrontendErrors.expected

@@ -0,0 +1 @@
+| empty-file.go:1:1:1:1 | expected 'package', found 'EOF' |

ql/test/query-tests/Summary/LinesOfCode.expected

@@ -0,0 +1 @@
+| 686 |

ql/test/query-tests/Summary/LinesOfCode.qlref

@@ -0,0 +1 @@
+Summary/LinesOfCode.ql

ql/test/query-tests/Summary/generated.go

@@ -0,0 +1,6 @@
+// Code generated for a test, DO NOT EDIT.
+package main
+
+func generated() {
+
+}

ql/test/query-tests/Summary/go.mod

@@ -0,0 +1,5 @@
+module codeql-go-tests/summary
+
+go 1.16
+
+require github.com/github/codeql-go v1.27.0 // indirect

ql/test/query-tests/Summary/short-file.go

@@ -0,0 +1,5 @@
+package main
+
+func main() {
+
+}

ql/test/query-tests/Summary/vendor/github.com/github/codeql-go/extractor/util/stub.go

@@ -0,0 +1,12 @@
+// Code generated by depstubber. DO NOT EDIT.
+// This is a simple stub for github.com/github/codeql-go/extractor/util, strictly for use in testing.
+
+// See the LICENSE file for information about the licensing of the original library.
+// Source: github.com/github/codeql-go/extractor/util (exports: ; functions: Getenv)
+
+// Package util is a stub of github.com/github/codeql-go/extractor/util, generated by depstubber.
+package util
+
+func Getenv(_ string, _ ...string) string {
+	return ""
+}

ql/test/query-tests/Summary/vendor/modules.txt

@@ -0,0 +1,3 @@
+# github.com/github/codeql-go v1.27.0
+## explicit
+github.com/github/codeql-go