Merge pull request #213 from sauyon/codeql-test

Use codeql for testing and add binary cross compilation support

.codeqlmanifest.json

@@ -1,5 +1,6 @@
 { "provide": [ "ql/src/qlpack.yml",
                "ql/test/qlpack.yml",
                "upgrades/qlpack.yml",
-               "ql/config/legacy-support/qlpack.yml" ],
+               "ql/config/legacy-support/qlpack.yml",
+               "build/codeql-extractor-go/codeql-extractor.yml" ],
   "ignore": [ "the-extractor-which-needs-to-be-built" ] }

.gitignore

@@ -18,4 +18,7 @@ ql/test/**/go.sum
 
 # binaries
 tools/bin
+tools/linux64
+tools/osx64
+tools/win64
 tools/tokenizer.jar

Makefile

@@ -2,26 +2,73 @@ all: tools ql/src/go.dbscheme
 
 ifeq ($(OS),Windows_NT)
 EXE = .exe
+CODEQL_PLATFORM = win64
 else
 EXE =
+UNAME_S := $(shell uname -s)
+ifeq ($(UNAME_S),Linux)
+CODEQL_PLATFORM = linux64
+endif
+ifeq ($(UNAME_S),Darwin)
+CODEQL_PLATFORM = osx64
+endif
 endif
 
-.PHONY: tools
+CODEQL_TOOLS = $(addprefix codeql-tools/,autobuild.cmd autobuild.sh index.cmd index.sh)
-tools: tools/bin/go-extractor$(EXE) tools/bin/go-tokenizer$(EXE) tools/bin/go-autobuilder$(EXE) tools/tokenizer.jar tools/bin/go-bootstrap$(EXE)
 
-tools/bin/go-extractor$(EXE): FORCE
+EXTRACTOR_PACK_OUT = build/codeql-extractor-go
-	go build -mod=vendor -o $@ ./extractor/cli/go-extractor
 
-tools/bin/go-tokenizer$(EXE): FORCE
+BINARIES = go-extractor go-tokenizer go-autobuilder go-bootstrap
-	go build -mod=vendor -o $@ ./extractor/cli/go-tokenizer
 
-tools/bin/go-autobuilder$(EXE): FORCE
+.PHONY: tools tools-codeql tools-codeql-full clean \
-	go build -mod=vendor -o $@ ./extractor/cli/go-autobuilder
+	tools-linux64 tools-osx64 tools-win64
 
-tools/bin/go-bootstrap$(EXE): FORCE
+clean:
-	go build -mod=vendor -o $@ ./extractor/cli/go-bootstrap
+	rm -rf tools/bin tools/linux64 tools/osx64 tools/win64 tools/net tools/opencsv
+	rm -rf $(EXTRACTOR_PACK_OUT) build/stats-project build/testdb
 
-FORCE:
+tools: $(addsuffix $(EXE),$(addprefix tools/bin/,$(BINARIES))) tools/tokenizer.jar
+
+$(addsuffix $(EXE),$(addprefix tools/bin/,$(BINARIES))):
+	go build -mod=vendor -o $@ ./extractor/cli/$(basename $(notdir $@))
+
+tools-codeql: tools-$(CODEQL_PLATFORM)
+
+tools-codeql-full: tools-linux64 tools-osx64 tools-win64
+
+tools-linux64: $(addprefix tools/linux64/,$(BINARIES)) tools/tokenizer.jar
+
+$(addprefix tools/linux64/,$(BINARIES)):
+	GOOS=linux GOARCH=amd64 go build -mod=vendor -o $@ ./extractor/cli/$(notdir $@)
+
+tools-osx64: $(addprefix tools/osx64/,$(BINARIES)) tools/tokenizer.jar
+
+$(addprefix tools/osx64/,$(BINARIES)):
+	GOOS=darwin GOARCH=amd64 go build -mod=vendor -o $@ ./extractor/cli/$(notdir $@)
+
+tools-win64: $(addsuffix .exe,$(addprefix tools/win64/,$(BINARIES))) tools/tokenizer.jar
+
+$(addsuffix .exe,$(addprefix tools/win64/,$(BINARIES))):
+	GOOS=windows GOARCH=amd64 go build -mod=vendor -o $@ ./extractor/cli/$(basename $(notdir $@))
+
+.PHONY: extractor extractor-full
+extractor: codeql-extractor.yml COPYRIGHT LICENSE ql/src/go.dbscheme \
+	tools/tokenizer.jar tools-codeql $(CODEQL_TOOLS)
+	rm -rf $(EXTRACTOR_PACK_OUT)
+	mkdir -p $(EXTRACTOR_PACK_OUT)
+	cp codeql-extractor.yml COPYRIGHT LICENSE ql/src/go.dbscheme ql/src/go.dbscheme.stats $(EXTRACTOR_PACK_OUT)
+	mkdir $(EXTRACTOR_PACK_OUT)/tools
+	cp -r $(CODEQL_TOOLS) $(EXTRACTOR_PACK_OUT)/tools
+	cp -r tools/tokenizer.jar tools/$(CODEQL_PLATFORM) $(EXTRACTOR_PACK_OUT)/tools
+
+extractor-full: codeql-extractor.yml COPYRIGHT LICENSE ql/src/go.dbscheme \
+	tools/tokenizer.jar tools-codeql-full $(CODEQL_TOOLS)
+	rm -rf $(EXTRACTOR_PACK_OUT)
+	mkdir -p $(EXTRACTOR_PACK_OUT)
+	cp codeql-extractor.yml COPYRIGHT LICENSE ql/src/go.dbscheme ql/src/go.dbscheme.stats $(EXTRACTOR_PACK_OUT)
+	mkdir $(EXTRACTOR_PACK_OUT)/tools
+	cp -r $(CODEQL_TOOLS) $(EXTRACTOR_PACK_OUT)/tools
+	cp -r tools/tokenizer.jar $(addprefix tools/,linux64 osx64 win64) $(EXTRACTOR_PACK_OUT)/tools
 
 tools/tokenizer.jar: tools/net/sourceforge/pmd/cpd/GoLanguage.class
 	jar cf $@ -C tools net
@@ -34,8 +81,8 @@ tools/net/sourceforge/pmd/cpd/GoLanguage.class: extractor/net/sourceforge/pmd/cp
 	rm tools/net/sourceforge/pmd/cpd/TokenEntry.class
 	rm tools/net/sourceforge/pmd/cpd/Tokenizer.class
 
-ql/src/go.dbscheme: tools/bin/go-extractor$(EXE)
+ql/src/go.dbscheme: tools/$(CODEQL_PLATFORM)/go-extractor$(EXE)
-	env TRAP_FOLDER=/tmp tools/bin/go-extractor --dbscheme $@
+	env TRAP_FOLDER=/tmp $^ --dbscheme $@
 
 ql/src/go.dbscheme.stats: ql/src/go.dbscheme
 	odasa createProject --force --template templates/project --threads 4 \
@@ -47,15 +94,15 @@ ql/src/go.dbscheme.stats: ql/src/go.dbscheme
 	odasa buildSnapshot --latest --project build/stats-project
 	odasa collectStats --dbscheme $^ --db build/stats-project/revision/working/db-go --outputFile $@
 
-test: all build/testdb/check-upgrade-path
+test: all extractor build/testdb/check-upgrade-path
-	odasa qltest --language go --library ql/src ql/test
+	codeql test run ql/test --search-path . --additional-packs .
 	cd extractor; go test -mod=vendor ./... | grep -vF "[no test files]"
 
 .PHONY: build/testdb/check-upgrade-path
 build/testdb/check-upgrade-path : build/testdb/go.dbscheme ql/src/go.dbscheme
-	odasa upgradeDatabase --db build/testdb --upgrade-packs upgrades
+	codeql dataset upgrade build/testdb --search-path upgrades
 	diff -q build/testdb/go.dbscheme ql/src/go.dbscheme
 
 build/testdb/go.dbscheme: upgrades/initial/go.dbscheme
 	echo >build/empty.trap
-	odasa cli --dbscheme upgrades/initial/go.dbscheme --import build/empty.trap --db build/testdb
+	codeql dataset import -S upgrades/initial/go.dbscheme build/testdb build/empty.trap