Merge pull request #566 from dbartol/dbartol/refactor

Refactor Go pack into separate library and query packs

.codeqlmanifest.json

@@ -1,4 +1,5 @@
 { "provide": [ "ql/src/qlpack.yml",
+               "ql/lib/qlpack.yml",
                "ql/examples/qlpack.yml",
                "ql/test/qlpack.yml",
                "upgrades/qlpack.yml",

.github/workflows/codeqltest.yml

@@ -98,6 +98,8 @@ jobs:
   test-win:
     name: Test Windows
     runs-on: windows-latest
+    env:
+      CODEQL_LOCK_MODE: verify
     steps:
     - name: Set up Go 1.17
       uses: actions/setup-go@v1

.gitignore

@@ -23,3 +23,6 @@ tools/osx64
 tools/win64
 tools/tokenizer.jar
 main
+
+# QL pack output directories
+.codeql

Makefile

@@ -1,4 +1,4 @@
-all: extractor ql/src/go.dbscheme
+all: extractor ql/lib/go.dbscheme install-deps
 
 ifeq ($(OS),Windows_NT)
 EXE = .exe
@@ -37,6 +37,9 @@ check-formatting:
 	find ql -iregex '.*\.qll?' -print0 | xargs -0 codeql query format --check-only
 	test -z "$$(find . -path '**/vendor' -prune -or -type f -iname '*.go' ! -empty -print0 | xargs -0 grep -L "//\s*autoformat-ignore" | xargs gofmt -l)"
 
+install-deps:
+	bash scripts/install-deps.sh $(CODEQL_LOCK_MODE)
+
 ifeq ($(QHELP_OUT_DIR),)
 # If not otherwise specified, compile qhelp to markdown in place
 QHELP_OUT_DIR := ql/src
@@ -73,12 +76,12 @@ tools-win64: $(addsuffix .exe,$(addprefix tools/win64/,$(BINARIES)))
 $(addsuffix .exe,$(addprefix tools/win64/,$(BINARIES))):
 	env GOOS=windows GOARCH=amd64 go build -mod=vendor -o $@ ./extractor/cli/$(basename $(@F))
 
-.PHONY: extractor-common extractor extractor-full
-extractor-common: codeql-extractor.yml LICENSE ql/src/go.dbscheme \
+.PHONY: extractor-common extractor extractor-full install-deps
+extractor-common: codeql-extractor.yml LICENSE ql/lib/go.dbscheme \
 	tools/tokenizer.jar $(CODEQL_TOOLS)
 	rm -rf $(EXTRACTOR_PACK_OUT)
 	mkdir -p $(EXTRACTOR_PACK_OUT)
-	cp codeql-extractor.yml LICENSE ql/src/go.dbscheme ql/src/go.dbscheme.stats $(EXTRACTOR_PACK_OUT)
+	cp codeql-extractor.yml LICENSE ql/lib/go.dbscheme ql/lib/go.dbscheme.stats $(EXTRACTOR_PACK_OUT)
 	mkdir $(EXTRACTOR_PACK_OUT)/tools
 	cp -r tools/tokenizer.jar $(CODEQL_TOOLS) $(EXTRACTOR_PACK_OUT)/tools
 
@@ -99,7 +102,7 @@ tools/net/sourceforge/pmd/cpd/GoLanguage.class: extractor/net/sourceforge/pmd/cp
 	rm tools/net/sourceforge/pmd/cpd/TokenEntry.class
 	rm tools/net/sourceforge/pmd/cpd/Tokenizer.class
 
-ql/src/go.dbscheme: tools/$(CODEQL_PLATFORM)/go-gen-dbscheme$(EXE)
+ql/lib/go.dbscheme: tools/$(CODEQL_PLATFORM)/go-gen-dbscheme$(EXE)
 	$< $@
 
 build/stats/src.stamp:
@@ -108,7 +111,7 @@ build/stats/src.stamp:
 	git -C $(@D)/src checkout 9b52d559c609 -q
 	touch $@
 
-ql/src/go.dbscheme.stats: ql/src/go.dbscheme build/stats/src.stamp extractor
+ql/lib/go.dbscheme.stats: ql/lib/go.dbscheme build/stats/src.stamp extractor
 	rm -rf build/stats/database
 	codeql database create -l go -s build/stats/src -j4 --search-path . build/stats/database
 	codeql dataset measure -o $@ build/stats/database/db-go
@@ -121,9 +124,9 @@ test: all build/testdb/check-upgrade-path
 	bash extractor-smoke-test/test.sh || (echo "Extractor smoke test FAILED"; exit 1)
 
 .PHONY: build/testdb/check-upgrade-path
-build/testdb/check-upgrade-path : build/testdb/go.dbscheme ql/src/go.dbscheme
+build/testdb/check-upgrade-path : build/testdb/go.dbscheme ql/lib/go.dbscheme
 	codeql dataset upgrade build/testdb --search-path upgrades
-	diff -q build/testdb/go.dbscheme ql/src/go.dbscheme
+	diff -q build/testdb/go.dbscheme ql/lib/go.dbscheme
 
 .PHONY: build/testdb/go.dbscheme
 build/testdb/go.dbscheme: upgrades/initial/go.dbscheme

README.md

@@ -17,7 +17,11 @@ repository](https://github.com/github/codeql).
 
 ## Installation
 
-Simply clone this repository. There are no external dependencies.
+Clone this repository.
+
+Run `scripts/install-deps.sh`. This will ensure that the necessary external CodeQL packs are
+downloaded to your machine. You will need to re-run this script whenever you pull new commits from
+the repo.
 
 If you want to use the CodeQL extension for Visual Studio Code, import this repository into your VS
 Code workspace.

ql/examples/qlpack.lock.yml

@@ -0,0 +1,4 @@
+---
+dependencies: {}
+compiled: false
+lockVersion: 1.0.0

ql/examples/qlpack.yml

@@ -1,3 +1,4 @@
-name: codeql-go-examples
-version: 0.0.0
-libraryPathDependencies: codeql-go
+name: codeql/go-examples
+version: 0.0.2
+dependencies:
+    codeql/go-all: ^0.0.2

ql/lib/qlpack.lock.yml

@@ -0,0 +1,4 @@
+---
+dependencies: {}
+compiled: false
+lockVersion: 1.0.0

ql/lib/qlpack.yml

@@ -0,0 +1,7 @@
+name: codeql/go-all
+version: 0.0.2
+dbscheme: go.dbscheme
+extractor: go
+library: true
+dependencies:
+    codeql/go-upgrades: ^0.0.2