From 3393588353b2751996cefc6d1f68c9132c891ce8 Mon Sep 17 00:00:00 2001 From: Sauyon Lee Date: Sun, 18 Apr 2021 22:49:46 -0700 Subject: [PATCH 1/9] Move concepts imports to Concepts.qll --- ql/src/go.qll | 1 - ql/src/semmle/go/Concepts.qll | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ql/src/go.qll b/ql/src/go.qll index 574268b2627..7ebaa8a0626 100644 --- a/ql/src/go.qll +++ b/ql/src/go.qll @@ -21,7 +21,6 @@ import semmle.go.StringOps import semmle.go.Types import semmle.go.Util import semmle.go.VariableWithFields -import semmle.go.concepts.HTTP import semmle.go.controlflow.BasicBlocks import semmle.go.controlflow.ControlFlowGraph import semmle.go.controlflow.IR diff --git a/ql/src/semmle/go/Concepts.qll b/ql/src/semmle/go/Concepts.qll index 4ffa06e3ba9..3ceb721c7ac 100644 --- a/ql/src/semmle/go/Concepts.qll +++ b/ql/src/semmle/go/Concepts.qll @@ -7,6 +7,8 @@ import go import semmle.go.dataflow.FunctionInputsAndOutputs +import semmle.go.concepts.HTTP + /** * A data-flow node that executes an operating system command, * for instance by spawning a new process. From 2a80a60468b11742db146f2cf933ae6d1ca2feaa Mon Sep 17 00:00:00 2001 From: Sauyon Lee Date: Sun, 18 Apr 2021 22:54:21 -0700 Subject: [PATCH 2/9] Add GeneratedFile concept --- ql/src/filters/ClassifyFiles.ql | 19 +------- ql/src/semmle/go/Concepts.qll | 2 +- ql/src/semmle/go/concepts/GeneratedFile.qll | 50 +++++++++++++++++++++ 3 files changed, 52 insertions(+), 19 deletions(-) create mode 100644 ql/src/semmle/go/concepts/GeneratedFile.qll diff --git a/ql/src/filters/ClassifyFiles.ql b/ql/src/filters/ClassifyFiles.ql index 792e9832ec2..f5ac1258375 100644 --- a/ql/src/filters/ClassifyFiles.ql +++ b/ql/src/filters/ClassifyFiles.ql @@ -8,17 +8,6 @@ import go -string generatorCommentRegex() { - result = "Generated By\\b.*\\bDo not edit" or - result = "This (file|class|interface|art[ei]fact) (was|is|(has been)) (?:auto[ -]?)?gener(e?)ated" or - result = "Any modifications to this file will be lost" or - result = - "This (file|class|interface|art[ei]fact) (was|is) (?:mechanically|automatically) generated" or - result = "The following code was (?:auto[ -]?)?generated (?:by|from)" or - result = "Autogenerated by Thrift" or - result = "(Code g|G)enerated from .* by ANTLR" -} - predicate classify(File f, string category) { // tests f instanceof TestFile and @@ -29,13 +18,7 @@ predicate classify(File f, string category) { category = "library" or // generated code - exists(Comment c | c.getFile() = f | - c.getText().regexpMatch("(?i).*\\b(" + concat(generatorCommentRegex(), "|") + ")\\b.*") - or - // regular expression recommended for Go code generators - // (https://golang.org/pkg/cmd/go/internal/generate/) - c.getText().regexpMatch("^\\s*Code generated .* DO NOT EDIT\\.\\s*$") - ) and + f instanceof GeneratedFile and category = "generated" } diff --git a/ql/src/semmle/go/Concepts.qll b/ql/src/semmle/go/Concepts.qll index 3ceb721c7ac..f3920c2ec5c 100644 --- a/ql/src/semmle/go/Concepts.qll +++ b/ql/src/semmle/go/Concepts.qll @@ -6,8 +6,8 @@ import go import semmle.go.dataflow.FunctionInputsAndOutputs - import semmle.go.concepts.HTTP +import semmle.go.concepts.GeneratedFile /** * A data-flow node that executes an operating system command, diff --git a/ql/src/semmle/go/concepts/GeneratedFile.qll b/ql/src/semmle/go/concepts/GeneratedFile.qll new file mode 100644 index 00000000000..c70d3104282 --- /dev/null +++ b/ql/src/semmle/go/concepts/GeneratedFile.qll @@ -0,0 +1,50 @@ +/** Provides a class for generated files. */ + +import go + +/** Provides a class for generated files. */ +module GeneratedFile { + /** + * A file that has been generated. + * + * Extend this class to model new APIs. If you want to refine existing API models, + * extend `GeneratedFile` instead. + */ + abstract class Range extends File { } + + private string generatorCommentRegex() { + result = "Generated By\\b.*\\bDo not edit" or + result = + "This (file|class|interface|art[ei]fact) (was|is|(has been)) (?:auto[ -]?)?gener(e?)ated" or + result = "Any modifications to this file will be lost" or + result = + "This (file|class|interface|art[ei]fact) (was|is) (?:mechanically|automatically) generated" or + result = "The following code was (?:auto[ -]?)?generated (?:by|from)" or + result = "Autogenerated by Thrift" or + result = "(Code g|G)enerated from .* by ANTLR" + } + + private class CommentHeuristicGeneratedFile extends Range { + CommentHeuristicGeneratedFile() { + exists(Comment c | c.getFile() = this | + c.getText().regexpMatch("(?i).*\\b(" + concat(generatorCommentRegex(), "|") + ")\\b.*") + or + // regular expression recommended for Go code generators + // (https://golang.org/pkg/cmd/go/internal/generate/) + c.getText().regexpMatch("^\\s*Code generated .* DO NOT EDIT\\.\\s*$") + ) + } + } +} + +/** + * A file that has been generated. + * + * Extend this class to refine existing API models. If you want to model new APIs, + * extend `GeneratedFile::Range` instead. + */ +class GeneratedFile extends File { + GeneratedFile::Range self; + + GeneratedFile() { this = self } +} From ed978e439f1e0c7ba7f582cad57895ea93adbfc2 Mon Sep 17 00:00:00 2001 From: Sauyon Lee Date: Sun, 18 Apr 2021 23:13:55 -0700 Subject: [PATCH 3/9] Add GoFile and move HtmlFile to Files.qll --- ql/src/semmle/go/Files.qll | 10 ++++++++++ ql/src/semmle/go/HTML.qll | 7 ------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/ql/src/semmle/go/Files.qll b/ql/src/semmle/go/Files.qll index a35e2677312..ed27c6b557b 100644 --- a/ql/src/semmle/go/Files.qll +++ b/ql/src/semmle/go/Files.qll @@ -264,3 +264,13 @@ class File extends ExtractedOrExternalFile { exists(this.getAChild()) } } + +/** A Go file. */ +class GoFile extends File { + GoFile() { this.getExtension() = "go" } +} + +/** An HTML file. */ +class HtmlFile extends File { + HtmlFile() { this.getExtension().regexpMatch("x?html?") } +} diff --git a/ql/src/semmle/go/HTML.qll b/ql/src/semmle/go/HTML.qll index 82c8724cd4b..f4fb773ca8e 100644 --- a/ql/src/semmle/go/HTML.qll +++ b/ql/src/semmle/go/HTML.qll @@ -3,13 +3,6 @@ import go module HTML { - /** - * An HTML file. - */ - class HtmlFile extends File { - HtmlFile() { this.getExtension().regexpMatch("x?html?") } - } - /** * An HTML element. * From fa5cb652d83cf287ace8c9c8fb871b0ee29cd762 Mon Sep 17 00:00:00 2001 From: Sauyon Lee Date: Tue, 20 Apr 2021 15:30:49 -0700 Subject: [PATCH 4/9] Fix lines of code counting --- extractor/extractor.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/extractor/extractor.go b/extractor/extractor.go index 996e9be8517..b057465b686 100644 --- a/extractor/extractor.go +++ b/extractor/extractor.go @@ -1623,11 +1623,19 @@ func extractNumLines(tw *trap.Writer, fileName string, ast *ast.File) { pos, tok, lit := s.Scan() if tok == token.EOF { break - } else if tok != token.ILLEGAL { + } else if tok != token.ILLEGAL && !(tok == token.SEMICOLON && lit == "\n") { + // specifically exclude newlines that are treated as semicolons tkStartLine := f.Position(pos).Line tkEndLine := tkStartLine + strings.Count(lit, "\n") if tkEndLine > lastCodeLine { - linesOfCode += tkEndLine - tkStartLine + 1 + if tkStartLine <= lastCodeLine { + // if the start line is the same as the last code line we've seen we don't want to double + // count it + // note tkStartLine < lastCodeLine should not be possible + linesOfCode += tkEndLine - lastCodeLine + } else { + linesOfCode += tkEndLine - tkStartLine + 1 + } lastCodeLine = tkEndLine } } From 9f85846980b8f00383418c661340d61628bb43d5 Mon Sep 17 00:00:00 2001 From: Sauyon Lee Date: Sun, 18 Apr 2021 23:16:55 -0700 Subject: [PATCH 5/9] Add lines of code summary query --- ql/src/Summary/LinesOfCode.ql | 11 + .../UnexpectedFrontendErrors.expected | 1 + .../query-tests/Summary/LinesOfCode.expected | 1 + ql/test/query-tests/Summary/LinesOfCode.qlref | 1 + ql/test/query-tests/Summary/empty-file.go | 0 ql/test/query-tests/Summary/generated.go | 6 + ql/test/query-tests/Summary/go.mod | 5 + ql/test/query-tests/Summary/long-file.go | 1359 +++++++++++++++++ ql/test/query-tests/Summary/short-file.go | 5 + .../github/codeql-go/extractor/util/stub.go | 12 + .../query-tests/Summary/vendor/modules.txt | 3 + 11 files changed, 1404 insertions(+) create mode 100644 ql/src/Summary/LinesOfCode.ql create mode 100644 ql/test/query-tests/Summary/CONSISTENCY/UnexpectedFrontendErrors.expected create mode 100644 ql/test/query-tests/Summary/LinesOfCode.expected create mode 100644 ql/test/query-tests/Summary/LinesOfCode.qlref create mode 100644 ql/test/query-tests/Summary/empty-file.go create mode 100644 ql/test/query-tests/Summary/generated.go create mode 100644 ql/test/query-tests/Summary/go.mod create mode 100644 ql/test/query-tests/Summary/long-file.go create mode 100644 ql/test/query-tests/Summary/short-file.go create mode 100644 ql/test/query-tests/Summary/vendor/github.com/github/codeql-go/extractor/util/stub.go create mode 100644 ql/test/query-tests/Summary/vendor/modules.txt diff --git a/ql/src/Summary/LinesOfCode.ql b/ql/src/Summary/LinesOfCode.ql new file mode 100644 index 00000000000..87d8321e20b --- /dev/null +++ b/ql/src/Summary/LinesOfCode.ql @@ -0,0 +1,11 @@ +/** + * @id go/summary/lines-of-code + * @name Total lines of Go code in the database + * @description The total number of lines of Go code across all extracted files, including auto-generated files. This is a useful metric of the size of a database. For all files that were seen during the build, this query counts the lines of code, excluding whitespace or comments. + * @kind metric + * @tags summary + */ + +import go + +select sum(GoFile f | | f.getNumberOfLinesOfCode()) diff --git a/ql/test/query-tests/Summary/CONSISTENCY/UnexpectedFrontendErrors.expected b/ql/test/query-tests/Summary/CONSISTENCY/UnexpectedFrontendErrors.expected new file mode 100644 index 00000000000..a17dae68734 --- /dev/null +++ b/ql/test/query-tests/Summary/CONSISTENCY/UnexpectedFrontendErrors.expected @@ -0,0 +1 @@ +| empty-file.go:1:1:1:1 | expected 'package', found 'EOF' | diff --git a/ql/test/query-tests/Summary/LinesOfCode.expected b/ql/test/query-tests/Summary/LinesOfCode.expected new file mode 100644 index 00000000000..728bb1f7996 --- /dev/null +++ b/ql/test/query-tests/Summary/LinesOfCode.expected @@ -0,0 +1 @@ +| 686 | diff --git a/ql/test/query-tests/Summary/LinesOfCode.qlref b/ql/test/query-tests/Summary/LinesOfCode.qlref new file mode 100644 index 00000000000..b60eb791722 --- /dev/null +++ b/ql/test/query-tests/Summary/LinesOfCode.qlref @@ -0,0 +1 @@ +Summary/LinesOfCode.ql diff --git a/ql/test/query-tests/Summary/empty-file.go b/ql/test/query-tests/Summary/empty-file.go new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ql/test/query-tests/Summary/generated.go b/ql/test/query-tests/Summary/generated.go new file mode 100644 index 00000000000..c2c750693f5 --- /dev/null +++ b/ql/test/query-tests/Summary/generated.go @@ -0,0 +1,6 @@ +// Code generated for a test, DO NOT EDIT. +package main + +func generated() { + +} diff --git a/ql/test/query-tests/Summary/go.mod b/ql/test/query-tests/Summary/go.mod new file mode 100644 index 00000000000..a76e5810575 --- /dev/null +++ b/ql/test/query-tests/Summary/go.mod @@ -0,0 +1,5 @@ +module codeql-go-tests/summary + +go 1.16 + +require github.com/github/codeql-go v1.27.0 // indirect diff --git a/ql/test/query-tests/Summary/long-file.go b/ql/test/query-tests/Summary/long-file.go new file mode 100644 index 00000000000..0a0658882e8 --- /dev/null +++ b/ql/test/query-tests/Summary/long-file.go @@ -0,0 +1,1359 @@ +package main + +import ( + "github.com/github/codeql-go/extractor/util" +) + +func aa() { + util.Getenv("environment") +} + +func ab() {} // comment + +func ac() {} + +func ad() {} + +func ae() {} + +func af() {} + +func ag() {} + +func ah() {} + +func ai() {} + +func aj() {} + +func ak() {} + +func al() {} + +func am() {} + +func an() {} + +func ao() {} + +func ap() {} + +func aq() {} + +func ar() {} + +func as() {} + +func at() {} + +func au() {} // more comments + +// comments over functions +func av() {} + +/* + * Block comment + */ +func aw() {} + +func ax() {} + +func ay() {} + +func az() {} + +func ba() {} + +func bb() {} + +func bc() {} + +func bd() {} + +func be() {} + +func bf() {} + +func bg() {} + +func bh() {} + +func bi() {} + +func bj() {} + +func bk() {} + +func bl() {} + +func bm() {} + +func bn() {} + +func bo() {} + +func bp() {} + +func bq() {} + +func br() {} + +func bs() {} + +func bt() {} + +func bu() {} + +func bv() {} + +func bw() {} + +func bx() {} + +func by() {} + +func bz() {} + +func ca() {} + +func cb() {} + +func cc() {} + +func cd() {} + +func ce() {} + +func cf() {} + +func cg() {} + +func ch() {} + +func ci() {} + +func cj() {} + +func ck() {} + +func cl() {} + +func cm() {} + +func cn() {} + +func co() {} + +func cp() {} + +func cq() {} + +func cr() {} + +func cs() {} + +func ct() {} + +func cu() {} + +func cv() {} + +func cw() {} + +func cx() {} + +func cy() {} + +func cz() {} + +func da() {} + +func db() {} + +func dc() {} + +func dd() {} + +func de() {} + +func df() {} + +func dg() {} + +func dh() {} + +func di() {} + +func dj() {} + +func dk() {} + +func dl() {} + +func dm() {} + +func dn() {} + +func do() {} + +func dp() {} + +func dq() {} + +func dr() {} + +func ds() {} + +func dt() {} + +func du() {} + +func dv() {} + +func dw() {} + +func dx() {} + +func dy() {} + +func dz() {} + +func ea() {} + +func eb() {} + +func ec() {} + +func ed() {} + +func ee() {} + +func ef() {} + +func eg() {} + +func eh() {} + +func ei() {} + +func ej() {} + +func ek() {} + +func el() {} + +func em() {} + +func en() {} + +func eo() {} + +func ep() {} + +func eq() {} + +func er() {} + +func es() {} + +func et() {} + +func eu() {} + +func ev() {} + +func ew() {} + +func ex() {} + +func ey() {} + +func ez() {} + +func fa() {} + +func fb() {} + +func fc() {} + +func fd() {} + +func fe() {} + +func ff() {} + +func fg() {} + +func fh() {} + +func fi() {} + +func fj() {} + +func fk() {} + +func fl() {} + +func fm() {} + +func fn() {} + +func fo() {} + +func fp() {} + +func fq() {} + +func fr() {} + +func fs() {} + +func ft() {} + +func fu() {} + +func fv() {} + +func fw() {} + +func fx() {} + +func fy() {} + +func fz() {} + +func ga() {} + +func gb() {} + +func gc() {} + +func gd() {} + +func ge() {} + +func gf() {} + +func gg() {} + +func gh() {} + +func gi() {} + +func gj() {} + +func gk() {} + +func gl() {} + +func gm() {} + +func gn() {} + +func gp() {} + +func gq() {} + +func gr() {} + +func gs() {} + +func gt() {} + +func gu() {} + +func gv() {} + +func gw() {} + +func gx() {} + +func gy() {} + +func gz() {} + +func ha() {} + +func hb() {} + +func hc() {} + +func hd() {} + +func he() {} + +func hf() {} + +func hg() {} + +func hh() {} + +func hi() {} + +func hj() {} + +func hk() {} + +func hl() {} + +func hm() {} + +func hn() {} + +func ho() {} + +func hp() {} + +func hq() {} + +func hr() {} + +func hs() {} + +func ht() {} + +func hu() {} + +func hv() {} + +func hw() {} + +func hx() {} + +func hy() {} + +func hz() {} + +func ia() {} + +func ib() {} + +func ic() {} + +func id() {} + +func ie() {} + +func ig() {} + +func ih() {} + +func ii() {} + +func ij() {} + +func ik() {} + +func il() {} + +func im() {} + +func in() {} + +func io() {} + +func ip() {} + +func iq() {} + +func ir() {} + +func is() {} + +func it() {} + +func iu() {} + +func iv() {} + +func iw() {} + +func ix() {} + +func iy() {} + +func iz() {} + +func ja() {} + +func jb() {} + +func jc() {} + +func jd() {} + +func je() {} + +func jf() {} + +func jg() {} + +func jh() {} + +func ji() {} + +func jj() {} + +func jk() {} + +func jl() {} + +func jm() {} + +func jn() {} + +func jo() {} + +func jp() {} + +func jq() {} + +func jr() {} + +func js() {} + +func jt() {} + +func ju() {} + +func jv() {} + +func jw() {} + +func jx() {} + +func jy() {} + +func jz() {} + +func ka() {} + +func kb() {} + +func kc() {} + +func kd() {} + +func ke() {} + +func kf() {} + +func kg() {} + +func kh() {} + +func ki() {} + +func kj() {} + +func kk() {} + +func kl() {} + +func km() {} + +func kn() {} + +func ko() {} + +func kp() {} + +func kq() {} + +func kr() {} + +func ks() {} + +func kt() {} + +func ku() {} + +func kv() {} + +func kw() {} + +func kx() {} + +func ky() {} + +func kz() {} + +func la() {} + +func lb() {} + +func lc() {} + +func ld() {} + +func le() {} + +func lf() {} + +func lg() {} + +func lh() {} + +func li() {} + +func lj() {} + +func lk() {} + +func ll() {} + +func lm() {} + +func ln() {} + +func lo() {} + +func lp() {} + +func lq() {} + +func lr() {} + +func ls() {} + +func lt() {} + +func lu() {} + +func lv() {} + +func lw() {} + +func lx() {} + +func ly() {} + +func lz() {} + +func ma() {} + +func mb() {} + +func mc() {} + +func md() {} + +func me() {} + +func mf() {} + +func mg() {} + +func mh() {} + +func mi() {} + +func mj() {} + +func mk() {} + +func ml() {} + +func mm() {} + +func mn() {} + +func mo() {} + +func mp() {} + +func mq() {} + +func mr() {} + +func ms() {} + +func mt() {} + +func mu() {} + +func mv() {} + +func mw() {} + +func mx() {} + +func my() {} + +func mz() {} + +func na() {} + +func nb() {} + +func nc() {} + +func nd() {} + +func ne() {} + +func nf() {} + +func ng() {} + +func nh() {} + +func ni() {} + +func nj() {} + +func nk() {} + +func nl() {} + +func nm() {} + +func nn() {} + +func no() {} + +func np() {} + +func nq() {} + +func nr() {} + +func ns() {} + +func nt() {} + +func nu() {} + +func nv() {} + +func nw() {} + +func nx() {} + +func ny() {} + +func nz() {} + +func oa() {} + +func ob() {} + +func oc() {} + +func od() {} + +func oe() {} + +func of() {} + +func og() {} + +func oh() {} + +func oi() {} + +func oj() {} + +func ok() {} + +func ol() {} + +func om() {} + +func on() {} + +func oo() {} + +func op() {} + +func oq() {} + +func or() {} + +func os() {} + +func ot() {} + +func ou() {} + +func ov() {} + +func ow() {} + +func ox() {} + +func oy() {} + +func oz() {} + +func pa() {} + +func pb() {} + +func pc() {} + +func pd() {} + +func pe() {} + +func pf() {} + +func pg() {} + +func ph() {} + +func pi() {} + +func pj() {} + +func pk() {} + +func pl() {} + +func pm() {} + +func pn() {} + +func po() {} + +func pp() {} + +func pq() {} + +func pr() {} + +func ps() {} + +func pt() {} + +func pu() {} + +func pv() {} + +func pw() {} + +func px() {} + +func py() {} + +func pz() {} + +func qa() {} + +func qb() {} + +func qc() {} + +func qd() {} + +func qe() {} + +func qf() {} + +func qg() {} + +func qh() {} + +func qi() {} + +func qj() {} + +func qk() {} + +func ql() {} + +func qm() {} + +func qn() {} + +func qo() {} + +func qp() {} + +func qq() {} + +func qr() {} + +func qs() {} + +func qt() {} + +func qu() {} + +func qv() {} + +func qw() {} + +func qx() {} + +func qy() {} + +func qz() {} + +func ra() {} + +func rb() {} + +func rc() {} + +func rd() {} + +func re() {} + +func rf() {} + +func rg() {} + +func rh() {} + +func ri() {} + +func rj() {} + +func rk() {} + +func rl() {} + +func rm() {} + +func rn() {} + +func ro() {} + +func rp() {} + +func rq() {} + +func rr() {} + +func rs() {} + +func rt() {} + +func ru() {} + +func rv() {} + +func rw() {} + +func rx() {} + +func ry() {} + +func rz() {} + +func sa() {} + +func sb() {} + +func sc() {} + +func sd() {} + +func se() {} + +func sf() {} + +func sg() {} + +func sh() {} + +func si() {} + +func sj() {} + +func sk() {} + +func sl() {} + +func sm() {} + +func sn() {} + +func so() {} + +func sp() {} + +func sq() {} + +func sr() {} + +func ss() {} + +func st() {} + +func su() {} + +func sv() {} + +func sw() {} + +func sx() {} + +func sy() {} + +func sz() {} + +func ta() {} + +func tb() {} + +func tc() {} + +func td() {} + +func te() {} + +func tf() {} + +func tg() {} + +func th() {} + +func ti() {} + +func tj() {} + +func tk() {} + +func tl() {} + +func tm() {} + +func tn() {} + +func to() {} + +func tp() {} + +func tq() {} + +func tr() {} + +func ts() {} + +func tt() {} + +func tu() {} + +func tv() {} + +func tw() {} + +func tx() {} + +func ty() {} + +func tz() {} + +func ua() {} + +func ub() {} + +func uc() {} + +func ud() {} + +func ue() {} + +func uf() {} + +func ug() {} + +func uh() {} + +func ui() {} + +func uj() {} + +func uk() {} + +func ul() {} + +func um() {} + +func un() {} + +func uo() {} + +func up() {} + +func uq() {} + +func ur() {} + +func us() {} + +func ut() {} + +func uu() {} + +func uv() {} + +func uw() {} + +func ux() {} + +func uy() {} + +func uz() {} + +func va() {} + +func vb() {} + +func vc() {} + +func vd() {} + +func ve() {} + +func vf() {} + +func vg() {} + +func vh() {} + +func vi() {} + +func vj() {} + +func vk() {} + +func vl() {} + +func vm() {} + +func vn() {} + +func vo() {} + +func vp() {} + +func vq() {} + +func vr() {} + +func vs() {} + +func vt() {} + +func vu() {} + +func vv() {} + +func vw() {} + +func vx() {} + +func vy() {} + +func vz() {} + +func wa() {} + +func wb() {} + +func wc() {} + +func wd() {} + +func we() {} + +func wf() {} + +func wg() {} + +func wh() {} + +func wi() {} + +func wj() {} + +func wk() {} + +func wl() {} + +func wm() {} + +func wn() {} + +func wo() {} + +func wp() {} + +func wq() {} + +func wr() {} + +func ws() {} + +func wt() {} + +func wu() {} + +func wv() {} + +func ww() {} + +func wx() {} + +func wy() {} + +func wz() {} + +func xa() {} + +func xb() {} + +func xc() {} + +func xd() {} + +func xe() {} + +func xf() {} + +func xg() {} + +func xh() {} + +func xi() {} + +func xj() {} + +func xk() {} + +func xl() {} + +func xm() {} + +func xn() {} + +func xo() {} + +func xp() {} + +func xq() {} + +func xr() {} + +func xs() {} + +func xt() {} + +func xu() {} + +func xv() {} + +func xw() {} + +func xx() {} + +func xy() {} + +func xz() {} + +func ya() {} + +func yb() {} + +func yc() {} + +func yd() {} + +func ye() {} + +func yf() {} + +func yg() {} + +func yh() {} + +func yi() {} + +func yj() {} + +func yk() {} + +func yl() {} + +func ym() {} + +func yn() {} + +func yo() {} + +func yp() {} + +func yq() {} + +func yr() {} + +func ys() {} + +func yt() {} + +func yu() {} + +func yv() {} + +func yw() {} + +func yx() {} + +func yy() {} + +func yz() {} + +func za() {} + +func zb() {} + +func zc() {} + +func zd() {} + +func ze() {} + +func zf() {} + +func zg() {} + +func zh() {} + +func zi() {} + +func zj() {} + +func zk() {} + +func zl() {} + +func zm() {} + +func zn() {} + +func zo() {} + +func zp() {} + +func zq() {} + +func zr() {} + +func zs() {} + +func zt() {} + +func zu() {} + +func zv() {} + +func zw() {} + +func zx() {} + +func zy() {} + +func zz() {} diff --git a/ql/test/query-tests/Summary/short-file.go b/ql/test/query-tests/Summary/short-file.go new file mode 100644 index 00000000000..79058077776 --- /dev/null +++ b/ql/test/query-tests/Summary/short-file.go @@ -0,0 +1,5 @@ +package main + +func main() { + +} diff --git a/ql/test/query-tests/Summary/vendor/github.com/github/codeql-go/extractor/util/stub.go b/ql/test/query-tests/Summary/vendor/github.com/github/codeql-go/extractor/util/stub.go new file mode 100644 index 00000000000..70d8e98f222 --- /dev/null +++ b/ql/test/query-tests/Summary/vendor/github.com/github/codeql-go/extractor/util/stub.go @@ -0,0 +1,12 @@ +// Code generated by depstubber. DO NOT EDIT. +// This is a simple stub for github.com/github/codeql-go/extractor/util, strictly for use in testing. + +// See the LICENSE file for information about the licensing of the original library. +// Source: github.com/github/codeql-go/extractor/util (exports: ; functions: Getenv) + +// Package util is a stub of github.com/github/codeql-go/extractor/util, generated by depstubber. +package util + +func Getenv(_ string, _ ...string) string { + return "" +} diff --git a/ql/test/query-tests/Summary/vendor/modules.txt b/ql/test/query-tests/Summary/vendor/modules.txt new file mode 100644 index 00000000000..704e904f224 --- /dev/null +++ b/ql/test/query-tests/Summary/vendor/modules.txt @@ -0,0 +1,3 @@ +# github.com/github/codeql-go v1.27.0 +## explicit +github.com/github/codeql-go From 27b72b53e5ace6aef00d0a380820042f086c8f97 Mon Sep 17 00:00:00 2001 From: Sauyon Lee Date: Mon, 26 Apr 2021 05:59:38 -0700 Subject: [PATCH 6/9] Add diagnostic queries --- ql/src/Diagnostics/DiagnosticsReporting.qll | 54 +++++++++++++++++++ ql/src/Diagnostics/ExtractionErrors.ql | 13 +++++ .../Diagnostics/SuccessfullyExtractedFiles.ql | 12 +++++ .../UnexpectedFrontendErrors.expected | 5 ++ .../Diagnostics/ExtractionErrors.expected | 4 ++ .../Diagnostics/ExtractionErrors.qlref | 1 + .../SuccessfullyExtractedFiles.expected | 1 + .../SuccessfullyExtractedFiles.qlref | 1 + ql/test/query-tests/Diagnostics/bad.go | 5 ++ ql/test/query-tests/Diagnostics/badimport.go | 7 +++ ql/test/query-tests/Diagnostics/badpkg.go | 14 +++++ .../Diagnostics/invalid{/invalid.go | 1 + ql/test/query-tests/Diagnostics/type.go | 12 +++++ ql/test/query-tests/Diagnostics/util.go | 5 ++ 14 files changed, 135 insertions(+) create mode 100644 ql/src/Diagnostics/DiagnosticsReporting.qll create mode 100644 ql/src/Diagnostics/ExtractionErrors.ql create mode 100644 ql/src/Diagnostics/SuccessfullyExtractedFiles.ql create mode 100644 ql/test/query-tests/Diagnostics/CONSISTENCY/UnexpectedFrontendErrors.expected create mode 100644 ql/test/query-tests/Diagnostics/ExtractionErrors.expected create mode 100644 ql/test/query-tests/Diagnostics/ExtractionErrors.qlref create mode 100644 ql/test/query-tests/Diagnostics/SuccessfullyExtractedFiles.expected create mode 100644 ql/test/query-tests/Diagnostics/SuccessfullyExtractedFiles.qlref create mode 100644 ql/test/query-tests/Diagnostics/bad.go create mode 100644 ql/test/query-tests/Diagnostics/badimport.go create mode 100644 ql/test/query-tests/Diagnostics/badpkg.go create mode 100644 ql/test/query-tests/Diagnostics/invalid{/invalid.go create mode 100644 ql/test/query-tests/Diagnostics/type.go create mode 100644 ql/test/query-tests/Diagnostics/util.go diff --git a/ql/src/Diagnostics/DiagnosticsReporting.qll b/ql/src/Diagnostics/DiagnosticsReporting.qll new file mode 100644 index 00000000000..3a6f1f18224 --- /dev/null +++ b/ql/src/Diagnostics/DiagnosticsReporting.qll @@ -0,0 +1,54 @@ +import go + +/** Gets the SARIF severity level that indicates an error. */ +private int getErrorSeverity() { result = 2 } + +private class Diagnostic extends @diagnostic { + /** + * Gets the kind of error. This can be: + * * `@unknownerror`: an unknown error + * * `@listerror`: an error from the frontend + * * `@parseerror`: a parse error + * * `@typeerror`: a type error + */ + string getKind() { diagnostics(this, _, result, _, _, _) } + + /** Gets the error message for this error. */ + string getMessage() { diagnostics(this, _, _, result, _, _) } + + /** Gets the file that this error is associated with, if any. */ + File getFile() { this.hasLocationInfo(result.getAbsolutePath(), _, _, _, _) } + + /** + * Holds if this element is at the specified location. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `filepath`. + * For more information, see + * [Locations](https://help.semmle.com/QL/learn-ql/ql/locations.html). + */ + predicate hasLocationInfo(string path, int sl, int sc, int el, int ec) { + exists(Location l | diagnostics(this, _, _, _, _, l) | l.hasLocationInfo(path, sl, sc, el, ec)) + } + + string toString() { result = this.getMessage() } +} + +/** + * Holds if an extraction error or warning occurred that should be reported to end users, + * with the error message `msg` and SARIF severity `sev`. + */ +predicate reportableDiagnostics(Diagnostic d, string msg, int sev) { + // Go does not have warnings, so all errors have error severity + sev = getErrorSeverity() and + ( + // Only report errors for files that would have been extracted + exists(File f | f = d.getFile() | + exists(f.getAChild()) and + msg = + "Extraction failed in " + d.getFile().getRelativePath() + " with error " + d.getMessage() + ) + or + not exists(d.getFile()) and + msg = "Extraction failed with error " + d.getMessage() + ) +} diff --git a/ql/src/Diagnostics/ExtractionErrors.ql b/ql/src/Diagnostics/ExtractionErrors.ql new file mode 100644 index 00000000000..921c22da1f2 --- /dev/null +++ b/ql/src/Diagnostics/ExtractionErrors.ql @@ -0,0 +1,13 @@ +/** + * @id go/diagnostics/extraction-errors + * @name Extraction errors + * @description List all extraction errors for files in the source code directory. + * @kind diagnostic + */ + +import go +import DiagnosticsReporting + +from string msg, int sev +where reportableDiagnostics(_, msg, sev) +select msg, sev diff --git a/ql/src/Diagnostics/SuccessfullyExtractedFiles.ql b/ql/src/Diagnostics/SuccessfullyExtractedFiles.ql new file mode 100644 index 00000000000..48f7103e813 --- /dev/null +++ b/ql/src/Diagnostics/SuccessfullyExtractedFiles.ql @@ -0,0 +1,12 @@ +/** + * @id go/summary/successfully-extracted-files + * @name Successfully analyzed files + * @description List all files that were successfully extracted. + * @kind diagnostic + */ + +import go + +from File f +where not exists(Error e | e.getFile() = f) +select f.getRelativePath() diff --git a/ql/test/query-tests/Diagnostics/CONSISTENCY/UnexpectedFrontendErrors.expected b/ql/test/query-tests/Diagnostics/CONSISTENCY/UnexpectedFrontendErrors.expected new file mode 100644 index 00000000000..e297b1b06ec --- /dev/null +++ b/ql/test/query-tests/Diagnostics/CONSISTENCY/UnexpectedFrontendErrors.expected @@ -0,0 +1,5 @@ +| ../Diagnostics:0:0:0:0 | malformed import path "github.com/github/codeql-go/ql/test/query-tests/Diagnostics/invalid{": invalid char '{' | +| bad.go:3:1:3:1 | expected 'package', found avvu | +| badpkg.go:4:2:4:2 | cannot find package "." in:\n\t/home/sauyon/devel/semmlego/vendor/github.com/github/nonexistent | +| badpkg.go:4:2:4:2 | could not import github.com/github/nonexistent (invalid package name: "") | +| type.go:11:9:11:9 | cannot use v (variable of type V) as T value in argument to takesT | diff --git a/ql/test/query-tests/Diagnostics/ExtractionErrors.expected b/ql/test/query-tests/Diagnostics/ExtractionErrors.expected new file mode 100644 index 00000000000..69ec0a68d85 --- /dev/null +++ b/ql/test/query-tests/Diagnostics/ExtractionErrors.expected @@ -0,0 +1,4 @@ +| Extraction failed in query-tests/Diagnostics/badpkg.go with error cannot find package "." in:\n\t/home/sauyon/devel/semmlego/vendor/github.com/github/nonexistent | 2 | +| Extraction failed in query-tests/Diagnostics/badpkg.go with error could not import github.com/github/nonexistent (invalid package name: "") | 2 | +| Extraction failed in query-tests/Diagnostics/type.go with error cannot use v (variable of type V) as T value in argument to takesT | 2 | +| Extraction failed with error expected 'package', found avvu | 2 | diff --git a/ql/test/query-tests/Diagnostics/ExtractionErrors.qlref b/ql/test/query-tests/Diagnostics/ExtractionErrors.qlref new file mode 100644 index 00000000000..488db09ab05 --- /dev/null +++ b/ql/test/query-tests/Diagnostics/ExtractionErrors.qlref @@ -0,0 +1 @@ +Diagnostics/ExtractionErrors.ql diff --git a/ql/test/query-tests/Diagnostics/SuccessfullyExtractedFiles.expected b/ql/test/query-tests/Diagnostics/SuccessfullyExtractedFiles.expected new file mode 100644 index 00000000000..1791a62090c --- /dev/null +++ b/ql/test/query-tests/Diagnostics/SuccessfullyExtractedFiles.expected @@ -0,0 +1 @@ +| query-tests/Diagnostics/util.go | diff --git a/ql/test/query-tests/Diagnostics/SuccessfullyExtractedFiles.qlref b/ql/test/query-tests/Diagnostics/SuccessfullyExtractedFiles.qlref new file mode 100644 index 00000000000..e67b23fd557 --- /dev/null +++ b/ql/test/query-tests/Diagnostics/SuccessfullyExtractedFiles.qlref @@ -0,0 +1 @@ +Diagnostics/SuccessfullyExtractedFiles.ql diff --git a/ql/test/query-tests/Diagnostics/bad.go b/ql/test/query-tests/Diagnostics/bad.go new file mode 100644 index 00000000000..baed29e6e1a --- /dev/null +++ b/ql/test/query-tests/Diagnostics/bad.go @@ -0,0 +1,5 @@ +// autoformat-ignore + +avvu + +wnvwun diff --git a/ql/test/query-tests/Diagnostics/badimport.go b/ql/test/query-tests/Diagnostics/badimport.go new file mode 100644 index 00000000000..82ba8600892 --- /dev/null +++ b/ql/test/query-tests/Diagnostics/badimport.go @@ -0,0 +1,7 @@ +// autoformat-ignore + +package main + +import ( + "github.com/pkg{}" +) diff --git a/ql/test/query-tests/Diagnostics/badpkg.go b/ql/test/query-tests/Diagnostics/badpkg.go new file mode 100644 index 00000000000..77fdde5d2b3 --- /dev/null +++ b/ql/test/query-tests/Diagnostics/badpkg.go @@ -0,0 +1,14 @@ +package main + +import ( + "github.com/github/nonexistent" +) + +func test0() { + var v nonexistent.T + nonexistent.F() + + w := nonexistent.K + + use(v, w) +} diff --git a/ql/test/query-tests/Diagnostics/invalid{/invalid.go b/ql/test/query-tests/Diagnostics/invalid{/invalid.go new file mode 100644 index 00000000000..d54466cba4a --- /dev/null +++ b/ql/test/query-tests/Diagnostics/invalid{/invalid.go @@ -0,0 +1 @@ +package invalid diff --git a/ql/test/query-tests/Diagnostics/type.go b/ql/test/query-tests/Diagnostics/type.go new file mode 100644 index 00000000000..42e423c44ce --- /dev/null +++ b/ql/test/query-tests/Diagnostics/type.go @@ -0,0 +1,12 @@ +package main + +type T int + +type V int + +func takesT(t T) {} + +func passesV() { + var v V + takesT(v) +} diff --git a/ql/test/query-tests/Diagnostics/util.go b/ql/test/query-tests/Diagnostics/util.go new file mode 100644 index 00000000000..b819083a175 --- /dev/null +++ b/ql/test/query-tests/Diagnostics/util.go @@ -0,0 +1,5 @@ +package main + +func use(_ ...interface{}) { + +} From 03c3b15caa04c0a777a80c5f0aa5316e77dfe7a2 Mon Sep 17 00:00:00 2001 From: Sauyon Lee Date: Mon, 26 Apr 2021 06:04:19 -0700 Subject: [PATCH 7/9] Improve autoformatting check --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index f9a145f16ea..7d22401e9b6 100644 --- a/Makefile +++ b/Makefile @@ -30,12 +30,12 @@ clean: DATAFLOW_BRANCH=main autoformat: - find ql -name "*.ql" -or -name "*.qll" | xargs codeql query format -qq -i - git ls-files | grep '\.go$$' | grep -v ^vendor/ | xargs grep -L "//\s*autoformat-ignore" | xargs gofmt -w + find ql -iregex '.*\.qll?' -print0 | xargs -0 codeql query format -qq -i + find . -path '**/vendor' -prune -or -type f -iname '*.go' ! -empty -print0 | xargs -0 grep -L "//\s*autoformat-ignore" | xargs gofmt -w check-formatting: - find ql -name "*.ql" -or -name "*.qll" | xargs codeql query format --check-only - test -z "$$(git ls-files | grep '\.go$$' | grep -v ^vendor/ | xargs grep -L "//\s*autoformat-ignore" | xargs gofmt -l)" + find ql -iregex '.*\.qll?' -print0 | xargs -0 codeql query format --check-only + test -z "$$(find . -path '**/vendor' -prune -or -type f -iname '*.go' ! -empty -print0 | xargs -0 grep -L "//\s*autoformat-ignore" | xargs gofmt -l)" ifeq ($(QHELP_OUT_DIR),) # If not otherwise specified, compile qhelp to markdown in place From d09cb7f22846c3afa80826fafd6d37d2b276d929 Mon Sep 17 00:00:00 2001 From: Sauyon Lee Date: Mon, 26 Apr 2021 23:58:45 -0700 Subject: [PATCH 8/9] Remove badpkg.go to make tests location-independent --- .../CONSISTENCY/UnexpectedFrontendErrors.expected | 2 -- .../Diagnostics/ExtractionErrors.expected | 2 -- ql/test/query-tests/Diagnostics/badpkg.go | 14 -------------- 3 files changed, 18 deletions(-) delete mode 100644 ql/test/query-tests/Diagnostics/badpkg.go diff --git a/ql/test/query-tests/Diagnostics/CONSISTENCY/UnexpectedFrontendErrors.expected b/ql/test/query-tests/Diagnostics/CONSISTENCY/UnexpectedFrontendErrors.expected index e297b1b06ec..71e5ed320fb 100644 --- a/ql/test/query-tests/Diagnostics/CONSISTENCY/UnexpectedFrontendErrors.expected +++ b/ql/test/query-tests/Diagnostics/CONSISTENCY/UnexpectedFrontendErrors.expected @@ -1,5 +1,3 @@ | ../Diagnostics:0:0:0:0 | malformed import path "github.com/github/codeql-go/ql/test/query-tests/Diagnostics/invalid{": invalid char '{' | | bad.go:3:1:3:1 | expected 'package', found avvu | -| badpkg.go:4:2:4:2 | cannot find package "." in:\n\t/home/sauyon/devel/semmlego/vendor/github.com/github/nonexistent | -| badpkg.go:4:2:4:2 | could not import github.com/github/nonexistent (invalid package name: "") | | type.go:11:9:11:9 | cannot use v (variable of type V) as T value in argument to takesT | diff --git a/ql/test/query-tests/Diagnostics/ExtractionErrors.expected b/ql/test/query-tests/Diagnostics/ExtractionErrors.expected index 69ec0a68d85..842899c1eef 100644 --- a/ql/test/query-tests/Diagnostics/ExtractionErrors.expected +++ b/ql/test/query-tests/Diagnostics/ExtractionErrors.expected @@ -1,4 +1,2 @@ -| Extraction failed in query-tests/Diagnostics/badpkg.go with error cannot find package "." in:\n\t/home/sauyon/devel/semmlego/vendor/github.com/github/nonexistent | 2 | -| Extraction failed in query-tests/Diagnostics/badpkg.go with error could not import github.com/github/nonexistent (invalid package name: "") | 2 | | Extraction failed in query-tests/Diagnostics/type.go with error cannot use v (variable of type V) as T value in argument to takesT | 2 | | Extraction failed with error expected 'package', found avvu | 2 | diff --git a/ql/test/query-tests/Diagnostics/badpkg.go b/ql/test/query-tests/Diagnostics/badpkg.go deleted file mode 100644 index 77fdde5d2b3..00000000000 --- a/ql/test/query-tests/Diagnostics/badpkg.go +++ /dev/null @@ -1,14 +0,0 @@ -package main - -import ( - "github.com/github/nonexistent" -) - -func test0() { - var v nonexistent.T - nonexistent.F() - - w := nonexistent.K - - use(v, w) -} From bfe6e7510d9402fca571ad582fd7c3502c23bb47 Mon Sep 17 00:00:00 2001 From: Sauyon Lee Date: Tue, 27 Apr 2021 08:28:56 -0700 Subject: [PATCH 9/9] Evaluate symlinks for the dummy file --- extractor/extractor.go | 12 +++++++++--- .../CONSISTENCY/UnexpectedFrontendErrors.expected | 2 +- .../Diagnostics/ExtractionErrors.expected | 1 + 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/extractor/extractor.go b/extractor/extractor.go index b057465b686..704f80ebbed 100644 --- a/extractor/extractor.go +++ b/extractor/extractor.go @@ -460,11 +460,17 @@ func (extraction *Extraction) extractError(tw *trap.Writer, err packages.Error, if pos == "" { // extract a dummy file - file, e = filepath.Abs(filepath.Join(".", "-")) + wd, e := os.Getwd() if e != nil { - file = filepath.Join(".", "-") - log.Printf("Warning: failed to get absolute path for for %s", file) + wd = "." + log.Printf("Warning: failed to get working directory") } + ewd, e := filepath.EvalSymlinks(wd) + if e != nil { + ewd = wd + log.Printf("Warning: failed to evaluate symlinks for %s", wd) + } + file = filepath.Join(ewd, "-") } else { var rawfile string if parts := threePartPos.FindStringSubmatch(pos); parts != nil { diff --git a/ql/test/query-tests/Diagnostics/CONSISTENCY/UnexpectedFrontendErrors.expected b/ql/test/query-tests/Diagnostics/CONSISTENCY/UnexpectedFrontendErrors.expected index 71e5ed320fb..dfd4a4ff15d 100644 --- a/ql/test/query-tests/Diagnostics/CONSISTENCY/UnexpectedFrontendErrors.expected +++ b/ql/test/query-tests/Diagnostics/CONSISTENCY/UnexpectedFrontendErrors.expected @@ -1,3 +1,3 @@ -| ../Diagnostics:0:0:0:0 | malformed import path "github.com/github/codeql-go/ql/test/query-tests/Diagnostics/invalid{": invalid char '{' | +| -:0:0:0:0 | malformed import path "github.com/github/codeql-go/ql/test/query-tests/Diagnostics/invalid{": invalid char '{' | | bad.go:3:1:3:1 | expected 'package', found avvu | | type.go:11:9:11:9 | cannot use v (variable of type V) as T value in argument to takesT | diff --git a/ql/test/query-tests/Diagnostics/ExtractionErrors.expected b/ql/test/query-tests/Diagnostics/ExtractionErrors.expected index 842899c1eef..4154baaaccd 100644 --- a/ql/test/query-tests/Diagnostics/ExtractionErrors.expected +++ b/ql/test/query-tests/Diagnostics/ExtractionErrors.expected @@ -1,2 +1,3 @@ | Extraction failed in query-tests/Diagnostics/type.go with error cannot use v (variable of type V) as T value in argument to takesT | 2 | | Extraction failed with error expected 'package', found avvu | 2 | +| Extraction failed with error malformed import path "github.com/github/codeql-go/ql/test/query-tests/Diagnostics/invalid{": invalid char '{' | 2 |