From 1eb9466de24fbed8d1783d83d9aac9fddd97c0de Mon Sep 17 00:00:00 2001 From: Sauyon Lee Date: Thu, 9 Jan 2020 11:08:15 -0800 Subject: [PATCH] Use codeql for testing and add binary cross compilation support Also add support for building the extractor inside this repository so that users can build and use the extractor, and an up-to-date version can be used for testing. --- .codeqlmanifest.json | 3 +- .gitignore | 3 ++ Makefile | 82 +++++++++++++++++++++++++++++++++++--------- 3 files changed, 70 insertions(+), 18 deletions(-) diff --git a/.codeqlmanifest.json b/.codeqlmanifest.json index aa0e9f383fd..306a11c6798 100644 --- a/.codeqlmanifest.json +++ b/.codeqlmanifest.json @@ -1,5 +1,6 @@ { "provide": [ "ql/src/qlpack.yml", "ql/test/qlpack.yml", "upgrades/qlpack.yml", - "ql/config/legacy-support/qlpack.yml" ], + "ql/config/legacy-support/qlpack.yml", + "build/codeql-extractor-go/codeql-extractor.yml" ], "ignore": [ "the-extractor-which-needs-to-be-built" ] } diff --git a/.gitignore b/.gitignore index bba12d84e03..e7067655aa3 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,7 @@ ql/test/**/go.sum # binaries tools/bin +tools/linux64 +tools/osx64 +tools/win64 tools/tokenizer.jar diff --git a/Makefile b/Makefile index ad2dcec7d9d..2d9af4e2e6f 100644 --- a/Makefile +++ b/Makefile @@ -2,26 +2,74 @@ all: tools ql/src/go.dbscheme ifeq ($(OS),Windows_NT) EXE = .exe +CODEQL_PLATFORM = win64 else EXE = +UNAME_S := $(shell uname -s) +ifeq ($(UNAME_S),Linux) +CODEQL_PLATFORM = linux64 +endif +ifeq ($(UNAME_S),Darwin) +CODEQL_PLATFORM = osx64 +endif endif -.PHONY: tools -tools: tools/bin/go-extractor$(EXE) tools/bin/go-tokenizer$(EXE) tools/bin/go-autobuilder$(EXE) tools/tokenizer.jar tools/bin/go-bootstrap$(EXE) +CODEQL_TOOLS = $(addprefix codeql-tools/,autobuild.cmd autobuild.sh index.cmd index.sh) -tools/bin/go-extractor$(EXE): FORCE - go build -mod=vendor -o $@ ./extractor/cli/go-extractor +EXTRACTOR_PACK_OUT = build/codeql-extractor-go -tools/bin/go-tokenizer$(EXE): FORCE - go build -mod=vendor -o $@ ./extractor/cli/go-tokenizer +BINARIES = go-extractor go-tokenizer go-autobuilder go-bootstrap -tools/bin/go-autobuilder$(EXE): FORCE - go build -mod=vendor -o $@ ./extractor/cli/go-autobuilder +.PHONY: tools tools-codeql tools-codeql-full clean \ + tools-linux64 tools-osx64 tools-win64 -tools/bin/go-bootstrap$(EXE): FORCE - go build -mod=vendor -o $@ ./extractor/cli/go-bootstrap +clean: + rm -rf tools/bin tools/linux64 tools/osx64 tools/win64 tools/net tools/opencsv + rm -rf $(EXTRACTOR_PACK_OUT) build/stats-project build/testdb/check-upgrade-path \ + build/testdb/go.dbscheme -FORCE: +tools: $(addprefix tools/bin/,$(BINARIES)) tools/tokenizer.jar + +$(addprefix tools/bin/,$(BINARIES)): + go build -mod=vendor -o $@ ./extractor/cli/$(notdir $@) + +tools-codeql: tools-$(CODEQL_PLATFORM) + +tools-codeql-full: tools-linux64 tools-osx64 tools-win64 + +tools-linux64: $(addprefix tools/linux64/,$(BINARIES)) tools/tokenizer.jar + +$(addprefix tools/linux64/,$(BINARIES)): + GOOS=linux GOARCH=amd64 go build -mod=vendor -o $@ ./extractor/cli/$(notdir $@) + +tools-osx64: $(addprefix tools/osx64/,$(BINARIES)) tools/tokenizer.jar + +$(addprefix tools/osx64/,$(BINARIES)): + GOOS=darwin GOARCH=amd64 go build -mod=vendor -o $@ ./extractor/cli/$(notdir $@) + +tools-win64: $(addsuffix .exe,$(addprefix tools/win64/,$(BINARIES))) tools/tokenizer.jar + +$(addsuffix .exe,$(addprefix tools/win64/,$(BINARIES))): + GOOS=windows GOARCH=amd64 go build -mod=vendor -o $@ ./extractor/cli/$(basename $(notdir $@)) + +.PHONY: extractor extractor-full +extractor: codeql-extractor.yml COPYRIGHT LICENSE ql/src/go.dbscheme \ + tools/tokenizer.jar tools-codeql $(CODEQL_TOOLS) + rm -rf $(EXTRACTOR_PACK_OUT) + mkdir -p $(EXTRACTOR_PACK_OUT) + cp codeql-extractor.yml COPYRIGHT LICENSE ql/src/go.dbscheme ql/src/go.dbscheme.stats $(EXTRACTOR_PACK_OUT) + mkdir $(EXTRACTOR_PACK_OUT)/tools + cp -r $(CODEQL_TOOLS) $(EXTRACTOR_PACK_OUT)/tools + cp -r tools/tokenizer.jar tools/$(CODEQL_PLATFORM) $(EXTRACTOR_PACK_OUT)/tools + +extractor-full: codeql-extractor.yml COPYRIGHT LICENSE ql/src/go.dbscheme \ + tools/tokenizer.jar tools-codeql-full $(CODEQL_TOOLS) + rm -rf $(EXTRACTOR_PACK_OUT) + mkdir -p $(EXTRACTOR_PACK_OUT) + cp codeql-extractor.yml COPYRIGHT LICENSE ql/src/go.dbscheme ql/src/go.dbscheme.stats $(EXTRACTOR_PACK_OUT) + mkdir $(EXTRACTOR_PACK_OUT)/tools + cp -r $(CODEQL_TOOLS) $(EXTRACTOR_PACK_OUT)/tools + cp -r tools/tokenizer.jar $(addprefix tools/,linux64 osx64 win64) $(EXTRACTOR_PACK_OUT)/tools tools/tokenizer.jar: tools/net/sourceforge/pmd/cpd/GoLanguage.class jar cf $@ -C tools net @@ -34,8 +82,8 @@ tools/net/sourceforge/pmd/cpd/GoLanguage.class: extractor/net/sourceforge/pmd/cp rm tools/net/sourceforge/pmd/cpd/TokenEntry.class rm tools/net/sourceforge/pmd/cpd/Tokenizer.class -ql/src/go.dbscheme: tools/bin/go-extractor$(EXE) - env TRAP_FOLDER=/tmp tools/bin/go-extractor --dbscheme $@ +ql/src/go.dbscheme: tools/$(CODEQL_PLATFORM)/go-extractor$(EXE) + env TRAP_FOLDER=/tmp $^ --dbscheme $@ ql/src/go.dbscheme.stats: ql/src/go.dbscheme odasa createProject --force --template templates/project --threads 4 \ @@ -47,15 +95,15 @@ ql/src/go.dbscheme.stats: ql/src/go.dbscheme odasa buildSnapshot --latest --project build/stats-project odasa collectStats --dbscheme $^ --db build/stats-project/revision/working/db-go --outputFile $@ -test: all build/testdb/check-upgrade-path - odasa qltest --language go --library ql/src ql/test +test: all extractor build/testdb/check-upgrade-path + codeql test run ql/test --search-path . --additional-packs . cd extractor; go test -mod=vendor ./... | grep -vF "[no test files]" .PHONY: build/testdb/check-upgrade-path build/testdb/check-upgrade-path : build/testdb/go.dbscheme ql/src/go.dbscheme - odasa upgradeDatabase --db build/testdb --upgrade-packs upgrades + codeql dataset upgrade build/testdb --search-path upgrades diff -q build/testdb/go.dbscheme ql/src/go.dbscheme build/testdb/go.dbscheme: upgrades/initial/go.dbscheme echo >build/empty.trap - odasa cli --dbscheme upgrades/initial/go.dbscheme --import build/empty.trap --db build/testdb + codeql dataset import -S upgrades/initial/go.dbscheme build/testdb build/empty.trap