diff --git a/.codeqlmanifest.json b/.codeqlmanifest.json index aa0e9f383fd..306a11c6798 100644 --- a/.codeqlmanifest.json +++ b/.codeqlmanifest.json @@ -1,5 +1,6 @@ { "provide": [ "ql/src/qlpack.yml", "ql/test/qlpack.yml", "upgrades/qlpack.yml", - "ql/config/legacy-support/qlpack.yml" ], + "ql/config/legacy-support/qlpack.yml", + "build/codeql-extractor-go/codeql-extractor.yml" ], "ignore": [ "the-extractor-which-needs-to-be-built" ] } diff --git a/.gitignore b/.gitignore index bba12d84e03..e7067655aa3 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,7 @@ ql/test/**/go.sum # binaries tools/bin +tools/linux64 +tools/osx64 +tools/win64 tools/tokenizer.jar diff --git a/Makefile b/Makefile index ad2dcec7d9d..099cf6367ad 100644 --- a/Makefile +++ b/Makefile @@ -2,26 +2,73 @@ all: tools ql/src/go.dbscheme ifeq ($(OS),Windows_NT) EXE = .exe +CODEQL_PLATFORM = win64 else EXE = +UNAME_S := $(shell uname -s) +ifeq ($(UNAME_S),Linux) +CODEQL_PLATFORM = linux64 +endif +ifeq ($(UNAME_S),Darwin) +CODEQL_PLATFORM = osx64 +endif endif -.PHONY: tools -tools: tools/bin/go-extractor$(EXE) tools/bin/go-tokenizer$(EXE) tools/bin/go-autobuilder$(EXE) tools/tokenizer.jar tools/bin/go-bootstrap$(EXE) +CODEQL_TOOLS = $(addprefix codeql-tools/,autobuild.cmd autobuild.sh index.cmd index.sh) -tools/bin/go-extractor$(EXE): FORCE - go build -mod=vendor -o $@ ./extractor/cli/go-extractor +EXTRACTOR_PACK_OUT = build/codeql-extractor-go -tools/bin/go-tokenizer$(EXE): FORCE - go build -mod=vendor -o $@ ./extractor/cli/go-tokenizer +BINARIES = go-extractor go-tokenizer go-autobuilder go-bootstrap -tools/bin/go-autobuilder$(EXE): FORCE - go build -mod=vendor -o $@ ./extractor/cli/go-autobuilder +.PHONY: tools tools-codeql tools-codeql-full clean \ + tools-linux64 tools-osx64 tools-win64 -tools/bin/go-bootstrap$(EXE): FORCE - go build -mod=vendor -o $@ ./extractor/cli/go-bootstrap +clean: + rm -rf tools/bin tools/linux64 tools/osx64 tools/win64 tools/net tools/opencsv + rm -rf $(EXTRACTOR_PACK_OUT) build/stats-project build/testdb -FORCE: +tools: $(addsuffix $(EXE),$(addprefix tools/bin/,$(BINARIES))) tools/tokenizer.jar + +$(addsuffix $(EXE),$(addprefix tools/bin/,$(BINARIES))): + go build -mod=vendor -o $@ ./extractor/cli/$(basename $(notdir $@)) + +tools-codeql: tools-$(CODEQL_PLATFORM) + +tools-codeql-full: tools-linux64 tools-osx64 tools-win64 + +tools-linux64: $(addprefix tools/linux64/,$(BINARIES)) tools/tokenizer.jar + +$(addprefix tools/linux64/,$(BINARIES)): + GOOS=linux GOARCH=amd64 go build -mod=vendor -o $@ ./extractor/cli/$(notdir $@) + +tools-osx64: $(addprefix tools/osx64/,$(BINARIES)) tools/tokenizer.jar + +$(addprefix tools/osx64/,$(BINARIES)): + GOOS=darwin GOARCH=amd64 go build -mod=vendor -o $@ ./extractor/cli/$(notdir $@) + +tools-win64: $(addsuffix .exe,$(addprefix tools/win64/,$(BINARIES))) tools/tokenizer.jar + +$(addsuffix .exe,$(addprefix tools/win64/,$(BINARIES))): + GOOS=windows GOARCH=amd64 go build -mod=vendor -o $@ ./extractor/cli/$(basename $(notdir $@)) + +.PHONY: extractor extractor-full +extractor: codeql-extractor.yml COPYRIGHT LICENSE ql/src/go.dbscheme \ + tools/tokenizer.jar tools-codeql $(CODEQL_TOOLS) + rm -rf $(EXTRACTOR_PACK_OUT) + mkdir -p $(EXTRACTOR_PACK_OUT) + cp codeql-extractor.yml COPYRIGHT LICENSE ql/src/go.dbscheme ql/src/go.dbscheme.stats $(EXTRACTOR_PACK_OUT) + mkdir $(EXTRACTOR_PACK_OUT)/tools + cp -r $(CODEQL_TOOLS) $(EXTRACTOR_PACK_OUT)/tools + cp -r tools/tokenizer.jar tools/$(CODEQL_PLATFORM) $(EXTRACTOR_PACK_OUT)/tools + +extractor-full: codeql-extractor.yml COPYRIGHT LICENSE ql/src/go.dbscheme \ + tools/tokenizer.jar tools-codeql-full $(CODEQL_TOOLS) + rm -rf $(EXTRACTOR_PACK_OUT) + mkdir -p $(EXTRACTOR_PACK_OUT) + cp codeql-extractor.yml COPYRIGHT LICENSE ql/src/go.dbscheme ql/src/go.dbscheme.stats $(EXTRACTOR_PACK_OUT) + mkdir $(EXTRACTOR_PACK_OUT)/tools + cp -r $(CODEQL_TOOLS) $(EXTRACTOR_PACK_OUT)/tools + cp -r tools/tokenizer.jar $(addprefix tools/,linux64 osx64 win64) $(EXTRACTOR_PACK_OUT)/tools tools/tokenizer.jar: tools/net/sourceforge/pmd/cpd/GoLanguage.class jar cf $@ -C tools net @@ -34,8 +81,8 @@ tools/net/sourceforge/pmd/cpd/GoLanguage.class: extractor/net/sourceforge/pmd/cp rm tools/net/sourceforge/pmd/cpd/TokenEntry.class rm tools/net/sourceforge/pmd/cpd/Tokenizer.class -ql/src/go.dbscheme: tools/bin/go-extractor$(EXE) - env TRAP_FOLDER=/tmp tools/bin/go-extractor --dbscheme $@ +ql/src/go.dbscheme: tools/$(CODEQL_PLATFORM)/go-extractor$(EXE) + env TRAP_FOLDER=/tmp $^ --dbscheme $@ ql/src/go.dbscheme.stats: ql/src/go.dbscheme odasa createProject --force --template templates/project --threads 4 \ @@ -47,15 +94,15 @@ ql/src/go.dbscheme.stats: ql/src/go.dbscheme odasa buildSnapshot --latest --project build/stats-project odasa collectStats --dbscheme $^ --db build/stats-project/revision/working/db-go --outputFile $@ -test: all build/testdb/check-upgrade-path - odasa qltest --language go --library ql/src ql/test +test: all extractor build/testdb/check-upgrade-path + codeql test run ql/test --search-path . --additional-packs . cd extractor; go test -mod=vendor ./... | grep -vF "[no test files]" .PHONY: build/testdb/check-upgrade-path build/testdb/check-upgrade-path : build/testdb/go.dbscheme ql/src/go.dbscheme - odasa upgradeDatabase --db build/testdb --upgrade-packs upgrades + codeql dataset upgrade build/testdb --search-path upgrades diff -q build/testdb/go.dbscheme ql/src/go.dbscheme build/testdb/go.dbscheme: upgrades/initial/go.dbscheme echo >build/empty.trap - odasa cli --dbscheme upgrades/initial/go.dbscheme --import build/empty.trap --db build/testdb + codeql dataset import -S upgrades/initial/go.dbscheme build/testdb build/empty.trap