Merge pull request #480 from sauyon/go116

Add preliminary support for go 1.16

.github/workflows/codeqltest.yml

@@ -7,10 +7,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Set up Go 1.14
+    - name: Set up Go 1.16
       uses: actions/setup-go@v1
       with:
-        go-version: 1.14
+        go-version: 1.16
       id: go
 
     - name: Set up CodeQL CLI
@@ -52,10 +52,10 @@ jobs:
     name: Test MacOS
     runs-on: macOS-latest
     steps:
-    - name: Set up Go 1.14
+    - name: Set up Go 1.16
       uses: actions/setup-go@v1
       with:
-        go-version: 1.14
+        go-version: 1.16
       id: go
 
     - name: Set up CodeQL CLI
@@ -85,10 +85,10 @@ jobs:
     name: Test Windows
     runs-on: windows-latest
     steps:
-    - name: Set up Go 1.14
+    - name: Set up Go 1.16
       uses: actions/setup-go@v1
       with:
-        go-version: 1.14
+        go-version: 1.16
       id: go
 
     - name: Set up CodeQL CLI

Makefile

@@ -115,7 +115,8 @@ ql/src/go.dbscheme.stats: ql/src/go.dbscheme build/stats/src.stamp extractor
 
 test: all build/testdb/check-upgrade-path
 	codeql test run ql/test --search-path . --consistency-queries ql/test/consistency
-	env GOARCH=386 codeql$(EXE) test run ql/test/query-tests/Security/CWE-681 --search-path . --consistency-queries ql/test/consistency
+  #	use GOOS=linux because GOOS=darwin GOARCH=386 is no longer supported
+	env GOOS=linux GOARCH=386 codeql$(EXE) test run ql/test/query-tests/Security/CWE-681 --search-path . --consistency-queries ql/test/consistency
 	cd extractor; go test -mod=vendor ./... | grep -vF "[no test files]"
 
 .PHONY: build/testdb/check-upgrade-path

change-notes/2021-02-18-go-116.md

@@ -0,0 +1,2 @@
+lgtm,codescanning
+* The extractor now supports Go 1.16 and the new `io/fs` library that was introduced.

extractor/cli/go-autobuilder/go-autobuilder.go

@@ -549,10 +549,13 @@ func main() {
 				install = exec.Command("glide", "install")
 				log.Println("Installing dependencies using `glide install`")
 			} else {
+				// explicitly set go module support
 				if depMode == GoGetWithModules {
-					// enable go modules if used
 					os.Setenv("GO111MODULE", "on")
+				} else if depMode == GoGetNoModules {
+					os.Setenv("GO111MODULE", "off")
 				}
+
 				// get dependencies
 				install = exec.Command("go", "get", "-v", "./...")
 				log.Println("Installing dependencies using `go get -v ./...`.")

go.mod

@@ -1,6 +1,6 @@
 module github.com/github/codeql-go
 
-go 1.14
+go 1.16
 
 require (
 	golang.org/x/mod v0.3.0

ql/src/semmle/go/frameworks/Stdlib.qll

@@ -40,6 +40,7 @@ import semmle.go.frameworks.stdlib.Fmt
 import semmle.go.frameworks.stdlib.Html
 import semmle.go.frameworks.stdlib.HtmlTemplate
 import semmle.go.frameworks.stdlib.Io
+import semmle.go.frameworks.stdlib.IoFs
 import semmle.go.frameworks.stdlib.IoIoutil
 import semmle.go.frameworks.stdlib.Log
 import semmle.go.frameworks.stdlib.Mime

ql/src/semmle/go/frameworks/stdlib/Io.qll

@@ -61,6 +61,14 @@ module Io {
       // signature: func WriteString(w Writer, s string) (n int, err error)
       hasQualifiedName("io", "WriteString") and
       (inp.isParameter(1) and outp.isParameter(0))
+      or
+      // signature: func NopCloser(r io.Reader) io.ReadCloser
+      hasQualifiedName("io", "NopCloser") and
+      (inp.isParameter(0) and outp.isResult())
+      or
+      // signature: func ReadAll(r io.Reader) ([]byte, error)
+      hasQualifiedName("io", "ReadAll") and
+      (inp.isParameter(0) and outp.isResult(0))
     }
 
     override predicate hasTaintFlow(FunctionInput input, FunctionOutput output) {

ql/src/semmle/go/frameworks/stdlib/IoFs.qll

@@ -0,0 +1,100 @@
+/**
+ * Provides classes modeling security-relevant aspects of the 'io/fs' package.
+ */
+
+import go
+
+/**
+ * Provides classes modeling security-relevant aspects of the 'io/fs' package.
+ */
+module IoFs {
+  /** Gets the package name `io/fs`. */
+  string packagePath() { result = "io/fs" }
+
+  private class FunctionModels extends TaintTracking::FunctionModel {
+    FunctionInput inp;
+    FunctionOutput outp;
+
+    FunctionModels() {
+      //signature: func Glob(fsys FS, pattern string) (matches []string, err error)
+      this.hasQualifiedName(packagePath(), "Glob") and
+      (inp.isParameter(0) and outp.isResult(0))
+      or
+      //signature: func ReadFile(fsys FS, name string) ([]byte, error)
+      this.hasQualifiedName(packagePath(), "ReadFile") and
+      (inp.isParameter(0) and outp.isResult(0))
+      or
+      //signature: func ReadDir(fsys FS, name string) ([]DirEntry, error)
+      this.hasQualifiedName(packagePath(), "ReadDir") and
+      (inp.isParameter(0) and outp.isResult(0))
+      or
+      //signature: func Sub(fsys FS, dir string) (FS, error)
+      this.hasQualifiedName(packagePath(), "Sub") and
+      (inp.isParameter(0) and outp.isResult(0))
+    }
+
+    override predicate hasTaintFlow(FunctionInput input, FunctionOutput output) {
+      input = inp and output = outp
+    }
+  }
+
+  /**
+   * Models a step from `fs` to `path` and `d` in
+   * `fs.WalkDir(fs, "root", func(path string, d DirEntry, err error) {}`
+   */
+  private class WalkDirStep extends TaintTracking::AdditionalTaintStep {
+    override predicate step(DataFlow::Node pred, DataFlow::Node succ) {
+      //signature: func WalkDir(fsys FS, root string, fn WalkDirFunc) error
+      exists(DataFlow::CallNode call, DataFlow::FunctionNode f |
+        call.getTarget().hasQualifiedName(packagePath(), "WalkDir") and
+        f.getASuccessor*() = call.getArgument(2)
+      |
+        pred = call.getArgument(0) and
+        succ = f.getParameter([0, 1])
+      )
+    }
+  }
+
+  private class MethodModels extends TaintTracking::FunctionModel, Method {
+    FunctionInput inp;
+    FunctionOutput outp;
+
+    MethodModels() {
+      //signature: func (DirEntry).Name() string
+      this.implements(packagePath(), "DirEntry", "Name") and
+      (inp.isReceiver() and outp.isResult())
+      or
+      //signature: func (DirEntry).Info() (FileInfo, error)
+      this.implements(packagePath(), "DirEntry", "Info") and
+      (inp.isReceiver() and outp.isResult(0))
+      or
+      //signature: func (FS).Open(name string) (File, error)
+      this.implements(packagePath(), "FS", "Open") and
+      (inp.isReceiver() and outp.isResult(0))
+      or
+      //signature: func (GlobFS).Glob(pattern string) ([]string, error)
+      this.implements(packagePath(), "GlobFS", "Glob") and
+      (inp.isReceiver() and outp.isResult(0))
+      or
+      //signature: func (ReadDirFS).ReadDir(name string) ([]DirEntry, error)
+      this.implements(packagePath(), "ReadDirFS", "ReadDir") and
+      (inp.isReceiver() and outp.isResult(0))
+      or
+      //signature: func (ReadFileFS).ReadFile(name string) ([]byte, error)
+      this.implements(packagePath(), "ReadFileFS", "ReadFile") and
+      (inp.isReceiver() and outp.isResult(0))
+      or
+      //signature: func (SubFS).Sub(dir string) (FS, error)
+      this.implements(packagePath(), "SubFS", "Sub") and
+      (inp.isReceiver() and outp.isResult(0))
+      or
+      //signature: func (File).Read([]byte) (int, error)
+      this.implements(packagePath(), "File", "Read") and
+      (inp.isReceiver() and outp.isParameter(0))
+    }
+
+    override predicate hasTaintFlow(FunctionInput input, FunctionOutput output) {
+      input = inp and output = outp
+    }
+  }
+}

ql/src/semmle/go/frameworks/stdlib/Os.qll

@@ -53,6 +53,18 @@ module Os {
         fn = "Symlink" and pathidx in [0 .. 1]
         or
         fn = "Truncate" and pathidx = 0
+        or
+        fn = "DirFS" and pathidx = 0
+        or
+        fn = "ReadDir" and pathidx = 0
+        or
+        fn = "ReadFile" and pathidx = 0
+        or
+        fn = "MkdirTemp" and pathidx in [0 .. 1]
+        or
+        fn = "CreateTemp" and pathidx in [0 .. 1]
+        or
+        fn = "WriteFile" and pathidx = 0
       )
     }
 

ql/src/semmle/go/security/StoredXssCustomizations.qll

@@ -42,16 +42,13 @@ module StoredXss {
   class FileNameSource extends Source {
     FileNameSource() {
       // the second parameter to a filepath.Walk function
-      exists(DataFlow::ParameterNode prm, DataFlow::FunctionNode f, DataFlow::CallNode walkCall |
-        prm = this and
-        f.getParameter(0) = prm
-      |
-        walkCall.getTarget().hasQualifiedName("path/filepath", "Walk") and
-        walkCall.getArgument(1) = f.getASuccessor*()
+      exists(DataFlow::FunctionNode f, Function walkFn | this = f.getParameter(0) |
+        walkFn.hasQualifiedName("path/filepath", ["Walk", "WalkDir"]) and
+        walkFn.getACall().getArgument(1) = f.getASuccessor*()
       )
       or
       // A call to os.FileInfo.Name
-      exists(Method m | m.implements("os", "FileInfo", "Name") |
+      exists(Method m | m.implements("io/fs", "FileInfo", "Name") |
         m = this.(DataFlow::CallNode).getTarget()
       )
     }

ql/test/extractor-tests/go1.16/embed.expected

@@ -0,0 +1 @@
+| embeddedfile.go:9:5:9:5 | e |

ql/test/extractor-tests/go1.16/embed.ql

@@ -0,0 +1,5 @@
+import go
+
+from Variable v
+where exists(v.getDeclaration())
+select v

ql/test/extractor-tests/go1.16/embeddedfile.go

@@ -0,0 +1,15 @@
+package main
+
+import (
+	_ "embed"
+	"fmt"
+)
+
+//go:embed file
+var e string = "hi"
+
+func main() {
+	fmt.Println(e)
+	e = "why"
+	fmt.Println(e)
+}

ql/test/extractor-tests/go1.16/file

@@ -0,0 +1,2 @@
+file
+contents

ql/test/library-tests/semmle/go/frameworks/StdlibTaintFlow/Io.go

@@ -302,4 +302,14 @@ func RunAllTaints_Io() {
 		out := TaintStepTest_IoWriterToWriteTo_B0I0O0(source)
 		sink(24, out)
 	}
+	{
+		source := newSource(25).(io.Reader)
+		out := io.NopCloser(source)
+		sink(25, out)
+	}
+	{
+		source := newSource(26).(io.Reader)
+		out, _ := io.ReadAll(source)
+		sink(26, out)
+	}
 }

ql/test/library-tests/semmle/go/frameworks/StdlibTaintFlow/IoFs.go

@@ -0,0 +1,97 @@
+package main
+
+import (
+	"io/fs"
+)
+
+func walkDirCallback(path string, d fs.DirEntry, _ error) error {
+	sink(14, path)
+	sink(15, d)
+	return nil
+}
+
+func steps() {
+	{
+		source := newSource(0).(fs.FS)
+		out, _ := fs.Glob(source, "*")
+		sink(0, out)
+	}
+	{
+		source := newSource(1).(fs.FS)
+		out, _ := fs.ReadFile(source, "filename")
+		sink(1, out)
+	}
+	{
+		source := newSource(2).(fs.FS)
+		out, _ := fs.ReadDir(source, "dirname")
+		sink(2, out)
+	}
+	{
+		source := newSource(3).(fs.FS)
+		out, _ := fs.Sub(source, "dirname")
+		sink(3, out)
+	}
+	{
+		source := newSource(4).(fs.FS)
+		fs.WalkDir(source, ".", func(_ string, d fs.DirEntry, _ error) error {
+			sink(4, d)
+			return nil
+		})
+	}
+	{
+		source := newSource(5).(fs.FS)
+		fs.WalkDir(source, ".", func(path string, _ fs.DirEntry, _ error) error {
+			sink(5, path)
+			return nil
+		})
+	}
+	{
+		source := newSource(6).(fs.DirEntry)
+		out := source.Name()
+		sink(6, out)
+	}
+	{
+		source := newSource(7).(fs.DirEntry)
+		out, _ := source.Info()
+		sink(7, out)
+	}
+	{
+		source := newSource(8).(fs.FS)
+		out, _ := source.Open("filename")
+		sink(8, out)
+	}
+	{
+		source := newSource(9).(fs.GlobFS)
+		out, _ := source.Glob("*")
+		sink(9, out)
+	}
+	{
+		source := newSource(10).(fs.ReadDirFS)
+		out, _ := source.ReadDir("dirname")
+		sink(10, out)
+	}
+	{
+		source := newSource(11).(fs.ReadFileFS)
+		out, _ := source.ReadFile("filename")
+		sink(11, out)
+	}
+	{
+		source := newSource(12).(fs.SubFS)
+		out, _ := source.Sub("dirname")
+		sink(12, out)
+	}
+	{
+		source := newSource(13).(fs.File)
+		var out []byte
+		source.Read(out)
+		sink(13, out)
+	}
+	{
+		source := newSource(14).(fs.FS)
+		fs.WalkDir(source, ".", walkDirCallback)
+	}
+	{
+		source := newSource(15).(fs.FS)
+		fs.WalkDir(source, ".", walkDirCallback)
+	}
+}

ql/test/library-tests/semmle/go/frameworks/StdlibTaintFlow/Os.go

@@ -5,6 +5,7 @@ package main
 import (
 	"os"
 	"syscall"
+	"time"
 )
 
 func TaintStepTest_OsExpand_B0I0O0(sourceCQL interface{}) interface{} {
@@ -21,7 +22,7 @@ func TaintStepTest_OsExpandEnv_B0I0O0(sourceCQL interface{}) interface{} {
 
 func TaintStepTest_OsNewFile_B0I0O0(sourceCQL interface{}) interface{} {
 	fromUintptr784 := sourceCQL.(uintptr)
-	intoFile957 := os.NewFile(fromUintptr784, "")
+	intoFile957 := os.NewFile(fromUintptr784, "") // $fsaccess=""
 	return intoFile957
 }
 
@@ -149,3 +150,34 @@ func RunAllTaints_Os() {
 		sink(11, out)
 	}
 }
+
+func fsAccesses() {
+	var path, path1, part string
+	var time time.Time
+	os.Chdir(path)                       // $fsaccess=path
+	os.Chmod(path, 0600)                 // $fsaccess=path
+	os.Chown(path, 1000, 1000)           // $fsaccess=path
+	os.Chtimes(path, time, time)         // $fsaccess=path
+	os.Create(path)                      // $fsaccess=path
+	os.Lchown(path, 1000, 1000)          // $fsaccess=path
+	os.Link(path, path1)                 // $fsaccess=path $fsaccess=path1
+	os.Lstat(path)                       // $fsaccess=path
+	os.Mkdir(path, 0600)                 // $fsaccess=path
+	os.MkdirAll(path, 0600)              // $fsaccess=path
+	os.NewFile(124, path)                // $fsaccess=path
+	os.Open(path)                        // $fsaccess=path
+	os.OpenFile(path, os.O_RDONLY, 0600) // $fsaccess=path
+	os.Readlink(path)                    // $fsaccess=path
+	os.Remove(path)                      // $fsaccess=path
+	os.RemoveAll(path)                   // $fsaccess=path
+	os.Rename(path, path1)               // $fsaccess=path $fsaccess=path1
+	os.Stat(path)                        // $fsaccess=path
+	os.Symlink(path, path1)              // $fsaccess=path $fsaccess=path1
+	os.Truncate(path, 1000)              // $fsaccess=path
+	os.DirFS(path)                       // $fsaccess=path
+	os.ReadDir(path)                     // $fsaccess=path
+	os.ReadFile(path)                    // $fsaccess=path
+	os.MkdirTemp(path, part)             // $fsaccess=path $fsaccess=part
+	os.CreateTemp(path, part)            // $fsaccess=path $fsaccess=part
+	os.WriteFile(path, []byte{}, 0600)   // $fsaccess=path
+}

ql/test/library-tests/semmle/go/frameworks/StdlibTaintFlow/test.ql

@@ -0,0 +1,17 @@
+import go
+import TestUtilities.InlineExpectationsTest
+
+class FileSystemAccessTest extends InlineExpectationsTest {
+  FileSystemAccessTest() { this = "FileSystemAccess" }
+
+  override string getARelevantTag() { result = "fsaccess" }
+
+  override predicate hasActualResult(string file, int line, string element, string tag, string value) {
+    exists(FileSystemAccess f |
+      f.hasLocationInfo(file, line, _, _, _) and
+      element = f.toString() and
+      value = f.getAPathArgument().toString() and
+      tag = "fsaccess"
+    )
+  }
+}

ql/test/query-tests/Security/CWE-079/StoredXss.expected

@@ -1,11 +1,15 @@
 edges
 | StoredXss.go:13:21:13:31 | call to Name : string | StoredXss.go:13:21:13:36 | ...+... |
-| stored.go:16:3:16:28 | ... := ...[0] : pointer type | stored.go:28:22:28:25 | name |
+| stored.go:18:3:18:28 | ... := ...[0] : pointer type | stored.go:30:22:30:25 | name |
+| stored.go:59:30:59:33 | definition of path : string | stored.go:61:22:61:25 | path |
 nodes
 | StoredXss.go:13:21:13:31 | call to Name : string | semmle.label | call to Name : string |
 | StoredXss.go:13:21:13:36 | ...+... | semmle.label | ...+... |
-| stored.go:16:3:16:28 | ... := ...[0] : pointer type | semmle.label | ... := ...[0] : pointer type |
-| stored.go:28:22:28:25 | name | semmle.label | name |
+| stored.go:18:3:18:28 | ... := ...[0] : pointer type | semmle.label | ... := ...[0] : pointer type |
+| stored.go:30:22:30:25 | name | semmle.label | name |
+| stored.go:59:30:59:33 | definition of path : string | semmle.label | definition of path : string |
+| stored.go:61:22:61:25 | path | semmle.label | path |
 #select
 | StoredXss.go:13:21:13:36 | ...+... | StoredXss.go:13:21:13:31 | call to Name : string | StoredXss.go:13:21:13:36 | ...+... | Stored cross-site scripting vulnerability due to $@. | StoredXss.go:13:21:13:31 | call to Name | stored value |
-| stored.go:28:22:28:25 | name | stored.go:16:3:16:28 | ... := ...[0] : pointer type | stored.go:28:22:28:25 | name | Stored cross-site scripting vulnerability due to $@. | stored.go:16:3:16:28 | ... := ...[0] | stored value |
+| stored.go:30:22:30:25 | name | stored.go:18:3:18:28 | ... := ...[0] : pointer type | stored.go:30:22:30:25 | name | Stored cross-site scripting vulnerability due to $@. | stored.go:18:3:18:28 | ... := ...[0] | stored value |
+| stored.go:61:22:61:25 | path | stored.go:59:30:59:33 | definition of path : string | stored.go:61:22:61:25 | path | Stored cross-site scripting vulnerability due to $@. | stored.go:59:30:59:33 | definition of path | stored value |

ql/test/query-tests/Security/CWE-079/stored.go

@@ -3,8 +3,10 @@ package main
 import (
 	"database/sql"
 	"io"
+	"io/fs"
 	"log"
 	"net/http"
+	"path/filepath"
 )
 
 var db *sql.DB
@@ -51,3 +53,13 @@ func storedserve2() {
 		}
 	})
 }
+
+func storedserve3() {
+	http.HandleFunc("/user", func(w http.ResponseWriter, r *http.Request) {
+		filepath.WalkDir(".", func(path string, _ fs.DirEntry, err error) error {
+			// BAD: filenames are considered to be untrusted
+			io.WriteString(w, path)
+			return nil
+		})
+	})
+}