From 1eb9466de24fbed8d1783d83d9aac9fddd97c0de Mon Sep 17 00:00:00 2001 From: Sauyon Lee Date: Thu, 9 Jan 2020 11:08:15 -0800 Subject: [PATCH 1/3] Use codeql for testing and add binary cross compilation support Also add support for building the extractor inside this repository so that users can build and use the extractor, and an up-to-date version can be used for testing. --- .codeqlmanifest.json | 3 +- .gitignore | 3 ++ Makefile | 82 +++++++++++++++++++++++++++++++++++--------- 3 files changed, 70 insertions(+), 18 deletions(-) diff --git a/.codeqlmanifest.json b/.codeqlmanifest.json index aa0e9f383fd..306a11c6798 100644 --- a/.codeqlmanifest.json +++ b/.codeqlmanifest.json @@ -1,5 +1,6 @@ { "provide": [ "ql/src/qlpack.yml", "ql/test/qlpack.yml", "upgrades/qlpack.yml", - "ql/config/legacy-support/qlpack.yml" ], + "ql/config/legacy-support/qlpack.yml", + "build/codeql-extractor-go/codeql-extractor.yml" ], "ignore": [ "the-extractor-which-needs-to-be-built" ] } diff --git a/.gitignore b/.gitignore index bba12d84e03..e7067655aa3 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,7 @@ ql/test/**/go.sum # binaries tools/bin +tools/linux64 +tools/osx64 +tools/win64 tools/tokenizer.jar diff --git a/Makefile b/Makefile index ad2dcec7d9d..2d9af4e2e6f 100644 --- a/Makefile +++ b/Makefile @@ -2,26 +2,74 @@ all: tools ql/src/go.dbscheme ifeq ($(OS),Windows_NT) EXE = .exe +CODEQL_PLATFORM = win64 else EXE = +UNAME_S := $(shell uname -s) +ifeq ($(UNAME_S),Linux) +CODEQL_PLATFORM = linux64 +endif +ifeq ($(UNAME_S),Darwin) +CODEQL_PLATFORM = osx64 +endif endif -.PHONY: tools -tools: tools/bin/go-extractor$(EXE) tools/bin/go-tokenizer$(EXE) tools/bin/go-autobuilder$(EXE) tools/tokenizer.jar tools/bin/go-bootstrap$(EXE) +CODEQL_TOOLS = $(addprefix codeql-tools/,autobuild.cmd autobuild.sh index.cmd index.sh) -tools/bin/go-extractor$(EXE): FORCE - go build -mod=vendor -o $@ ./extractor/cli/go-extractor +EXTRACTOR_PACK_OUT = build/codeql-extractor-go -tools/bin/go-tokenizer$(EXE): FORCE - go build -mod=vendor -o $@ ./extractor/cli/go-tokenizer +BINARIES = go-extractor go-tokenizer go-autobuilder go-bootstrap -tools/bin/go-autobuilder$(EXE): FORCE - go build -mod=vendor -o $@ ./extractor/cli/go-autobuilder +.PHONY: tools tools-codeql tools-codeql-full clean \ + tools-linux64 tools-osx64 tools-win64 -tools/bin/go-bootstrap$(EXE): FORCE - go build -mod=vendor -o $@ ./extractor/cli/go-bootstrap +clean: + rm -rf tools/bin tools/linux64 tools/osx64 tools/win64 tools/net tools/opencsv + rm -rf $(EXTRACTOR_PACK_OUT) build/stats-project build/testdb/check-upgrade-path \ + build/testdb/go.dbscheme -FORCE: +tools: $(addprefix tools/bin/,$(BINARIES)) tools/tokenizer.jar + +$(addprefix tools/bin/,$(BINARIES)): + go build -mod=vendor -o $@ ./extractor/cli/$(notdir $@) + +tools-codeql: tools-$(CODEQL_PLATFORM) + +tools-codeql-full: tools-linux64 tools-osx64 tools-win64 + +tools-linux64: $(addprefix tools/linux64/,$(BINARIES)) tools/tokenizer.jar + +$(addprefix tools/linux64/,$(BINARIES)): + GOOS=linux GOARCH=amd64 go build -mod=vendor -o $@ ./extractor/cli/$(notdir $@) + +tools-osx64: $(addprefix tools/osx64/,$(BINARIES)) tools/tokenizer.jar + +$(addprefix tools/osx64/,$(BINARIES)): + GOOS=darwin GOARCH=amd64 go build -mod=vendor -o $@ ./extractor/cli/$(notdir $@) + +tools-win64: $(addsuffix .exe,$(addprefix tools/win64/,$(BINARIES))) tools/tokenizer.jar + +$(addsuffix .exe,$(addprefix tools/win64/,$(BINARIES))): + GOOS=windows GOARCH=amd64 go build -mod=vendor -o $@ ./extractor/cli/$(basename $(notdir $@)) + +.PHONY: extractor extractor-full +extractor: codeql-extractor.yml COPYRIGHT LICENSE ql/src/go.dbscheme \ + tools/tokenizer.jar tools-codeql $(CODEQL_TOOLS) + rm -rf $(EXTRACTOR_PACK_OUT) + mkdir -p $(EXTRACTOR_PACK_OUT) + cp codeql-extractor.yml COPYRIGHT LICENSE ql/src/go.dbscheme ql/src/go.dbscheme.stats $(EXTRACTOR_PACK_OUT) + mkdir $(EXTRACTOR_PACK_OUT)/tools + cp -r $(CODEQL_TOOLS) $(EXTRACTOR_PACK_OUT)/tools + cp -r tools/tokenizer.jar tools/$(CODEQL_PLATFORM) $(EXTRACTOR_PACK_OUT)/tools + +extractor-full: codeql-extractor.yml COPYRIGHT LICENSE ql/src/go.dbscheme \ + tools/tokenizer.jar tools-codeql-full $(CODEQL_TOOLS) + rm -rf $(EXTRACTOR_PACK_OUT) + mkdir -p $(EXTRACTOR_PACK_OUT) + cp codeql-extractor.yml COPYRIGHT LICENSE ql/src/go.dbscheme ql/src/go.dbscheme.stats $(EXTRACTOR_PACK_OUT) + mkdir $(EXTRACTOR_PACK_OUT)/tools + cp -r $(CODEQL_TOOLS) $(EXTRACTOR_PACK_OUT)/tools + cp -r tools/tokenizer.jar $(addprefix tools/,linux64 osx64 win64) $(EXTRACTOR_PACK_OUT)/tools tools/tokenizer.jar: tools/net/sourceforge/pmd/cpd/GoLanguage.class jar cf $@ -C tools net @@ -34,8 +82,8 @@ tools/net/sourceforge/pmd/cpd/GoLanguage.class: extractor/net/sourceforge/pmd/cp rm tools/net/sourceforge/pmd/cpd/TokenEntry.class rm tools/net/sourceforge/pmd/cpd/Tokenizer.class -ql/src/go.dbscheme: tools/bin/go-extractor$(EXE) - env TRAP_FOLDER=/tmp tools/bin/go-extractor --dbscheme $@ +ql/src/go.dbscheme: tools/$(CODEQL_PLATFORM)/go-extractor$(EXE) + env TRAP_FOLDER=/tmp $^ --dbscheme $@ ql/src/go.dbscheme.stats: ql/src/go.dbscheme odasa createProject --force --template templates/project --threads 4 \ @@ -47,15 +95,15 @@ ql/src/go.dbscheme.stats: ql/src/go.dbscheme odasa buildSnapshot --latest --project build/stats-project odasa collectStats --dbscheme $^ --db build/stats-project/revision/working/db-go --outputFile $@ -test: all build/testdb/check-upgrade-path - odasa qltest --language go --library ql/src ql/test +test: all extractor build/testdb/check-upgrade-path + codeql test run ql/test --search-path . --additional-packs . cd extractor; go test -mod=vendor ./... | grep -vF "[no test files]" .PHONY: build/testdb/check-upgrade-path build/testdb/check-upgrade-path : build/testdb/go.dbscheme ql/src/go.dbscheme - odasa upgradeDatabase --db build/testdb --upgrade-packs upgrades + codeql dataset upgrade build/testdb --search-path upgrades diff -q build/testdb/go.dbscheme ql/src/go.dbscheme build/testdb/go.dbscheme: upgrades/initial/go.dbscheme echo >build/empty.trap - odasa cli --dbscheme upgrades/initial/go.dbscheme --import build/empty.trap --db build/testdb + codeql dataset import -S upgrades/initial/go.dbscheme build/testdb build/empty.trap From 52fe0afa483f26d087985ea396831f4ab23c23c6 Mon Sep 17 00:00:00 2001 From: Sauyon Lee Date: Thu, 16 Jan 2020 16:21:09 -0800 Subject: [PATCH 2/3] Makefile: Delete entire test db in clean --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 2d9af4e2e6f..e24ad0516d0 100644 --- a/Makefile +++ b/Makefile @@ -25,8 +25,7 @@ BINARIES = go-extractor go-tokenizer go-autobuilder go-bootstrap clean: rm -rf tools/bin tools/linux64 tools/osx64 tools/win64 tools/net tools/opencsv - rm -rf $(EXTRACTOR_PACK_OUT) build/stats-project build/testdb/check-upgrade-path \ - build/testdb/go.dbscheme + rm -rf $(EXTRACTOR_PACK_OUT) build/stats-project build/testdb tools: $(addprefix tools/bin/,$(BINARIES)) tools/tokenizer.jar From 32fa033a55898f2df129b5ee62a6f43484dfa5f6 Mon Sep 17 00:00:00 2001 From: Sauyon Lee Date: Fri, 17 Jan 2020 14:01:09 -0800 Subject: [PATCH 3/3] Makefile: Add exe suffix back to tools/bin targets --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index e24ad0516d0..099cf6367ad 100644 --- a/Makefile +++ b/Makefile @@ -27,10 +27,10 @@ clean: rm -rf tools/bin tools/linux64 tools/osx64 tools/win64 tools/net tools/opencsv rm -rf $(EXTRACTOR_PACK_OUT) build/stats-project build/testdb -tools: $(addprefix tools/bin/,$(BINARIES)) tools/tokenizer.jar +tools: $(addsuffix $(EXE),$(addprefix tools/bin/,$(BINARIES))) tools/tokenizer.jar -$(addprefix tools/bin/,$(BINARIES)): - go build -mod=vendor -o $@ ./extractor/cli/$(notdir $@) +$(addsuffix $(EXE),$(addprefix tools/bin/,$(BINARIES))): + go build -mod=vendor -o $@ ./extractor/cli/$(basename $(notdir $@)) tools-codeql: tools-$(CODEQL_PLATFORM)