hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-01-29 14:23:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

Use codeql for testing and add binary cross compilation support

Browse Source 
Also add support for building the extractor inside this repository
so that users can build and use the extractor, and an up-to-date
version can be used for testing.
This commit is contained in:
Sauyon Lee
2020-01-09 11:08:15 -08:00
parent 2d97b396b7
commit 1eb9466de2
3 changed files with 70 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.codeqlmanifest.json
View File

@@ -1,5 +1,6 @@
{ "provide": [ "ql/src/qlpack.yml",
"ql/test/qlpack.yml",
"upgrades/qlpack.yml",
"ql/config/legacy-support/qlpack.yml" ],
"ql/config/legacy-support/qlpack.yml",
"build/codeql-extractor-go/codeql-extractor.yml" ],
"ignore": [ "the-extractor-which-needs-to-be-built" ] }

3
.gitignore vendored
View File

@@ -18,4 +18,7 @@ ql/test/**/go.sum
# binaries
tools/bin
tools/linux64
tools/osx64
tools/win64
tools/tokenizer.jar

82
Makefile
View File

@@ -2,26 +2,74 @@ all: tools ql/src/go.dbscheme
ifeq ($(OS),Windows_NT)
EXE = .exe
CODEQL_PLATFORM = win64
else
EXE =
UNAME_S := $(shell uname -s)
ifeq ($(UNAME_S),Linux)
CODEQL_PLATFORM = linux64
endif
ifeq ($(UNAME_S),Darwin)
CODEQL_PLATFORM = osx64
endif
endif
.PHONY: tools
tools: tools/bin/go-extractor$(EXE) tools/bin/go-tokenizer$(EXE) tools/bin/go-autobuilder$(EXE) tools/tokenizer.jar tools/bin/go-bootstrap$(EXE)
CODEQL_TOOLS = $(addprefix codeql-tools/,autobuild.cmd autobuild.sh index.cmd index.sh)
tools/bin/go-extractor$(EXE): FORCE
go build -mod=vendor -o $@ ./extractor/cli/go-extractor
EXTRACTOR_PACK_OUT = build/codeql-extractor-go
tools/bin/go-tokenizer$(EXE): FORCE
go build -mod=vendor -o $@ ./extractor/cli/go-tokenizer
BINARIES = go-extractor go-tokenizer go-autobuilder go-bootstrap
tools/bin/go-autobuilder$(EXE): FORCE
go build -mod=vendor -o $@ ./extractor/cli/go-autobuilder
.PHONY: tools tools-codeql tools-codeql-full clean \
tools-linux64 tools-osx64 tools-win64
tools/bin/go-bootstrap$(EXE): FORCE
go build -mod=vendor -o $@ ./extractor/cli/go-bootstrap
clean:
rm -rf tools/bin tools/linux64 tools/osx64 tools/win64 tools/net tools/opencsv
rm -rf $(EXTRACTOR_PACK_OUT) build/stats-project build/testdb/check-upgrade-path \
build/testdb/go.dbscheme
FORCE:
tools: $(addprefix tools/bin/,$(BINARIES)) tools/tokenizer.jar
$(addprefix tools/bin/,$(BINARIES)):
go build -mod=vendor -o $@ ./extractor/cli/$(notdir $@)
tools-codeql: tools-$(CODEQL_PLATFORM)
tools-codeql-full: tools-linux64 tools-osx64 tools-win64
tools-linux64: $(addprefix tools/linux64/,$(BINARIES)) tools/tokenizer.jar
$(addprefix tools/linux64/,$(BINARIES)):
GOOS=linux GOARCH=amd64 go build -mod=vendor -o $@ ./extractor/cli/$(notdir $@)
tools-osx64: $(addprefix tools/osx64/,$(BINARIES)) tools/tokenizer.jar
$(addprefix tools/osx64/,$(BINARIES)):
GOOS=darwin GOARCH=amd64 go build -mod=vendor -o $@ ./extractor/cli/$(notdir $@)
tools-win64: $(addsuffix .exe,$(addprefix tools/win64/,$(BINARIES))) tools/tokenizer.jar
$(addsuffix .exe,$(addprefix tools/win64/,$(BINARIES))):
GOOS=windows GOARCH=amd64 go build -mod=vendor -o $@ ./extractor/cli/$(basename $(notdir $@))
.PHONY: extractor extractor-full
extractor: codeql-extractor.yml COPYRIGHT LICENSE ql/src/go.dbscheme \
tools/tokenizer.jar tools-codeql $(CODEQL_TOOLS)
rm -rf $(EXTRACTOR_PACK_OUT)
mkdir -p $(EXTRACTOR_PACK_OUT)
cp codeql-extractor.yml COPYRIGHT LICENSE ql/src/go.dbscheme ql/src/go.dbscheme.stats $(EXTRACTOR_PACK_OUT)
mkdir $(EXTRACTOR_PACK_OUT)/tools
cp -r $(CODEQL_TOOLS) $(EXTRACTOR_PACK_OUT)/tools
cp -r tools/tokenizer.jar tools/$(CODEQL_PLATFORM) $(EXTRACTOR_PACK_OUT)/tools
extractor-full: codeql-extractor.yml COPYRIGHT LICENSE ql/src/go.dbscheme \
tools/tokenizer.jar tools-codeql-full $(CODEQL_TOOLS)
rm -rf $(EXTRACTOR_PACK_OUT)
mkdir -p $(EXTRACTOR_PACK_OUT)
cp codeql-extractor.yml COPYRIGHT LICENSE ql/src/go.dbscheme ql/src/go.dbscheme.stats $(EXTRACTOR_PACK_OUT)
mkdir $(EXTRACTOR_PACK_OUT)/tools
cp -r $(CODEQL_TOOLS) $(EXTRACTOR_PACK_OUT)/tools
cp -r tools/tokenizer.jar $(addprefix tools/,linux64 osx64 win64) $(EXTRACTOR_PACK_OUT)/tools
tools/tokenizer.jar: tools/net/sourceforge/pmd/cpd/GoLanguage.class
jar cf $@ -C tools net
@@ -34,8 +82,8 @@ tools/net/sourceforge/pmd/cpd/GoLanguage.class: extractor/net/sourceforge/pmd/cp
rm tools/net/sourceforge/pmd/cpd/TokenEntry.class
rm tools/net/sourceforge/pmd/cpd/Tokenizer.class
ql/src/go.dbscheme: tools/bin/go-extractor$(EXE)
env TRAP_FOLDER=/tmp tools/bin/go-extractor --dbscheme $@
ql/src/go.dbscheme: tools/$(CODEQL_PLATFORM)/go-extractor$(EXE)
env TRAP_FOLDER=/tmp $^ --dbscheme $@
ql/src/go.dbscheme.stats: ql/src/go.dbscheme
odasa createProject --force --template templates/project --threads 4 \
@@ -47,15 +95,15 @@ ql/src/go.dbscheme.stats: ql/src/go.dbscheme
odasa buildSnapshot --latest --project build/stats-project
odasa collectStats --dbscheme $^ --db build/stats-project/revision/working/db-go --outputFile $@
test: all build/testdb/check-upgrade-path
odasa qltest --language go --library ql/src ql/test
test: all extractor build/testdb/check-upgrade-path
codeql test run ql/test --search-path . --additional-packs .
cd extractor; go test -mod=vendor ./... | grep -vF "[no test files]"
.PHONY: build/testdb/check-upgrade-path
build/testdb/check-upgrade-path : build/testdb/go.dbscheme ql/src/go.dbscheme
odasa upgradeDatabase --db build/testdb --upgrade-packs upgrades
codeql dataset upgrade build/testdb --search-path upgrades
diff -q build/testdb/go.dbscheme ql/src/go.dbscheme
build/testdb/go.dbscheme: upgrades/initial/go.dbscheme
echo >build/empty.trap
odasa cli --dbscheme upgrades/initial/go.dbscheme --import build/empty.trap --db build/testdb
codeql dataset import -S upgrades/initial/go.dbscheme build/testdb build/empty.trap