From 393f6b76664ee13517cd9c9cee918867f3432d1b Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 24 Apr 2024 17:04:57 +0200 Subject: [PATCH 01/41] Go: add gazelle-generated `BUILD` files --- .pre-commit-config.yaml | 7 +++++ MODULE.bazel | 7 +++++ go/extractor/.gitattributes | 1 + go/extractor/BUILD.bazel | 23 ++++++++++++++++ go/extractor/autobuilder/BUILD.bazel | 24 +++++++++++++++++ go/extractor/cli/go-autobuilder/BUILD.bazel | 22 +++++++++++++++ go/extractor/cli/go-bootstrap/BUILD.bazel | 14 ++++++++++ go/extractor/cli/go-build-runner/BUILD.bazel | 18 +++++++++++++ go/extractor/cli/go-extractor/BUILD.bazel | 18 +++++++++++++ go/extractor/cli/go-gen-dbscheme/BUILD.bazel | 15 +++++++++++ go/extractor/cli/go-tokenizer/BUILD.bazel | 14 ++++++++++ go/extractor/dbscheme/BUILD.bazel | 15 +++++++++++ go/extractor/diagnostics/BUILD.bazel | 8 ++++++ go/extractor/project/BUILD.bazel | 21 +++++++++++++++ go/extractor/srcarchive/BUILD.bazel | 17 ++++++++++++ go/extractor/toolchain/BUILD.bazel | 18 +++++++++++++ go/extractor/trap/BUILD.bazel | 23 ++++++++++++++++ go/extractor/util/BUILD.bazel | 14 ++++++++++ .../x/mod/internal/lazyregexp/BUILD.bazel | 9 +++++++ .../golang.org/x/mod/modfile/BUILD.bazel | 19 +++++++++++++ .../golang.org/x/mod/module/BUILD.bazel | 16 +++++++++++ .../golang.org/x/mod/semver/BUILD.bazel | 9 +++++++ .../x/tools/go/gcexportdata/BUILD.bazel | 13 +++++++++ .../go/internal/packagesdriver/BUILD.bazel | 10 +++++++ .../x/tools/go/packages/BUILD.bazel | 25 +++++++++++++++++ .../x/tools/go/types/objectpath/BUILD.bazel | 10 +++++++ .../x/tools/internal/event/BUILD.bazel | 17 ++++++++++++ .../x/tools/internal/event/core/BUILD.bazel | 17 ++++++++++++ .../x/tools/internal/event/keys/BUILD.bazel | 14 ++++++++++ .../x/tools/internal/event/label/BUILD.bazel | 9 +++++++ .../x/tools/internal/event/tag/BUILD.bazel | 10 +++++++ .../x/tools/internal/gcimporter/BUILD.bazel | 27 +++++++++++++++++++ .../x/tools/internal/gocommand/BUILD.bazel | 20 ++++++++++++++ .../internal/packagesinternal/BUILD.bazel | 9 +++++++ .../x/tools/internal/pkgbits/BUILD.bazel | 21 +++++++++++++++ .../tools/internal/tokeninternal/BUILD.bazel | 9 +++++++ .../x/tools/internal/typeparams/BUILD.bazel | 15 +++++++++++ .../tools/internal/typesinternal/BUILD.bazel | 14 ++++++++++ .../x/tools/internal/versions/BUILD.bazel | 15 +++++++++++ go/gazelle/BUILD.bazel | 8 ++++++ go/rules.bzl | 0 41 files changed, 595 insertions(+) create mode 100644 go/extractor/.gitattributes create mode 100644 go/extractor/BUILD.bazel create mode 100644 go/extractor/autobuilder/BUILD.bazel create mode 100644 go/extractor/cli/go-autobuilder/BUILD.bazel create mode 100644 go/extractor/cli/go-bootstrap/BUILD.bazel create mode 100644 go/extractor/cli/go-build-runner/BUILD.bazel create mode 100644 go/extractor/cli/go-extractor/BUILD.bazel create mode 100644 go/extractor/cli/go-gen-dbscheme/BUILD.bazel create mode 100644 go/extractor/cli/go-tokenizer/BUILD.bazel create mode 100644 go/extractor/dbscheme/BUILD.bazel create mode 100644 go/extractor/diagnostics/BUILD.bazel create mode 100644 go/extractor/project/BUILD.bazel create mode 100644 go/extractor/srcarchive/BUILD.bazel create mode 100644 go/extractor/toolchain/BUILD.bazel create mode 100644 go/extractor/trap/BUILD.bazel create mode 100644 go/extractor/util/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/mod/internal/lazyregexp/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/mod/modfile/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/mod/module/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/mod/semver/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/go/gcexportdata/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/go/internal/packagesdriver/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/go/packages/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/go/types/objectpath/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/internal/event/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/internal/event/core/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/internal/event/keys/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/internal/event/label/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/internal/event/tag/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/internal/gcimporter/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/internal/gocommand/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/internal/packagesinternal/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/internal/pkgbits/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/internal/tokeninternal/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/internal/typeparams/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/internal/typesinternal/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/internal/versions/BUILD.bazel create mode 100644 go/gazelle/BUILD.bazel create mode 100644 go/rules.bzl diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 383bc110383..4e81bcc7711 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,6 +27,13 @@ repos: - repo: local hooks: + - id: gazelle + name: Check gazelle-generated BUILD files + files: go/extractor/.* + language: system + entry: bazel run //go/gazelle + pass_filenames: false + - id: codeql-format name: Fix QL file formatting files: \.qll?$ diff --git a/MODULE.bazel b/MODULE.bazel index 4e1fe0d9f7c..2dc9dd0a7bb 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -14,6 +14,7 @@ local_path_override( # see https://registry.bazel.build/ for a list of available packages bazel_dep(name = "platforms", version = "0.0.8") +bazel_dep(name = "rules_go", version = "0.47.0") bazel_dep(name = "rules_pkg", version = "0.10.1") bazel_dep(name = "rules_nodejs", version = "6.0.3") bazel_dep(name = "rules_python", version = "0.31.0") @@ -22,6 +23,8 @@ bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "absl") bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "json") bazel_dep(name = "fmt", version = "10.0.0") +bazel_dep(name = "gazelle", version = "0.36.0", dev_dependency = True) + pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") pip.parse( hub_name = "codegen_deps", @@ -50,6 +53,10 @@ node.toolchain( ) use_repo(node, "nodejs", "nodejs_toolchains") +go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk") +go_sdk.download(version = "1.22.2") +go_sdk.host() + register_toolchains( "@nodejs_toolchains//:all", ) diff --git a/go/extractor/.gitattributes b/go/extractor/.gitattributes new file mode 100644 index 00000000000..e406d413677 --- /dev/null +++ b/go/extractor/.gitattributes @@ -0,0 +1 @@ +/*/**/BUILD.bazel linguist-generated=true diff --git a/go/extractor/BUILD.bazel b/go/extractor/BUILD.bazel new file mode 100644 index 00000000000..ea99340e7d7 --- /dev/null +++ b/go/extractor/BUILD.bazel @@ -0,0 +1,23 @@ +load("@rules_go//go:def.bzl", "go_library") + +# gazelle:prefix github.com/github/codeql-go/extractor + +go_library( + name = "extractor", + srcs = [ + "extractor.go", + "gomodextractor.go", + "semaphore.go", + ], + importpath = "github.com/github/codeql-go/extractor", + visibility = ["//visibility:public"], + deps = [ + "//go/extractor/dbscheme", + "//go/extractor/diagnostics", + "//go/extractor/srcarchive", + "//go/extractor/trap", + "//go/extractor/util", + "//go/extractor/vendor/golang.org/x/mod/modfile", + "//go/extractor/vendor/golang.org/x/tools/go/packages", + ], +) diff --git a/go/extractor/autobuilder/BUILD.bazel b/go/extractor/autobuilder/BUILD.bazel new file mode 100644 index 00000000000..b81b15816aa --- /dev/null +++ b/go/extractor/autobuilder/BUILD.bazel @@ -0,0 +1,24 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "autobuilder", + srcs = [ + "autobuilder.go", + "build-environment.go", + ], + importpath = "github.com/github/codeql-go/extractor/autobuilder", + visibility = ["//visibility:public"], + deps = [ + "//go/extractor/diagnostics", + "//go/extractor/project", + "//go/extractor/toolchain", + "//go/extractor/util", + "//go/extractor/vendor/golang.org/x/mod/semver", + ], +) + +go_test( + name = "autobuilder_test", + srcs = ["build-environment_test.go"], + embed = [":autobuilder"], +) diff --git a/go/extractor/cli/go-autobuilder/BUILD.bazel b/go/extractor/cli/go-autobuilder/BUILD.bazel new file mode 100644 index 00000000000..b53227b9f1b --- /dev/null +++ b/go/extractor/cli/go-autobuilder/BUILD.bazel @@ -0,0 +1,22 @@ +load("@rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go-autobuilder_lib", + srcs = ["go-autobuilder.go"], + importpath = "github.com/github/codeql-go/extractor/cli/go-autobuilder", + visibility = ["//visibility:private"], + deps = [ + "//go/extractor/autobuilder", + "//go/extractor/diagnostics", + "//go/extractor/project", + "//go/extractor/toolchain", + "//go/extractor/util", + "//go/extractor/vendor/golang.org/x/mod/semver", + ], +) + +go_binary( + name = "go-autobuilder", + embed = [":go-autobuilder_lib"], + visibility = ["//visibility:public"], +) diff --git a/go/extractor/cli/go-bootstrap/BUILD.bazel b/go/extractor/cli/go-bootstrap/BUILD.bazel new file mode 100644 index 00000000000..15255227f9a --- /dev/null +++ b/go/extractor/cli/go-bootstrap/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go-bootstrap_lib", + srcs = ["go-bootstrap.go"], + importpath = "github.com/github/codeql-go/extractor/cli/go-bootstrap", + visibility = ["//visibility:private"], +) + +go_binary( + name = "go-bootstrap", + embed = [":go-bootstrap_lib"], + visibility = ["//visibility:public"], +) diff --git a/go/extractor/cli/go-build-runner/BUILD.bazel b/go/extractor/cli/go-build-runner/BUILD.bazel new file mode 100644 index 00000000000..fc407c3e172 --- /dev/null +++ b/go/extractor/cli/go-build-runner/BUILD.bazel @@ -0,0 +1,18 @@ +load("@rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go-build-runner_lib", + srcs = ["go-build-runner.go"], + importpath = "github.com/github/codeql-go/extractor/cli/go-build-runner", + visibility = ["//visibility:private"], + deps = [ + "//go/extractor/autobuilder", + "//go/extractor/util", + ], +) + +go_binary( + name = "go-build-runner", + embed = [":go-build-runner_lib"], + visibility = ["//visibility:public"], +) diff --git a/go/extractor/cli/go-extractor/BUILD.bazel b/go/extractor/cli/go-extractor/BUILD.bazel new file mode 100644 index 00000000000..9419c6f8ba9 --- /dev/null +++ b/go/extractor/cli/go-extractor/BUILD.bazel @@ -0,0 +1,18 @@ +load("@rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go-extractor_lib", + srcs = ["go-extractor.go"], + importpath = "github.com/github/codeql-go/extractor/cli/go-extractor", + visibility = ["//visibility:private"], + deps = [ + "//go/extractor", + "//go/extractor/diagnostics", + ], +) + +go_binary( + name = "go-extractor", + embed = [":go-extractor_lib"], + visibility = ["//visibility:public"], +) diff --git a/go/extractor/cli/go-gen-dbscheme/BUILD.bazel b/go/extractor/cli/go-gen-dbscheme/BUILD.bazel new file mode 100644 index 00000000000..1fbd75998c2 --- /dev/null +++ b/go/extractor/cli/go-gen-dbscheme/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go-gen-dbscheme_lib", + srcs = ["go-gen-dbscheme.go"], + importpath = "github.com/github/codeql-go/extractor/cli/go-gen-dbscheme", + visibility = ["//visibility:private"], + deps = ["//go/extractor/dbscheme"], +) + +go_binary( + name = "go-gen-dbscheme", + embed = [":go-gen-dbscheme_lib"], + visibility = ["//visibility:public"], +) diff --git a/go/extractor/cli/go-tokenizer/BUILD.bazel b/go/extractor/cli/go-tokenizer/BUILD.bazel new file mode 100644 index 00000000000..f55baecc69f --- /dev/null +++ b/go/extractor/cli/go-tokenizer/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go-tokenizer_lib", + srcs = ["go-tokenizer.go"], + importpath = "github.com/github/codeql-go/extractor/cli/go-tokenizer", + visibility = ["//visibility:private"], +) + +go_binary( + name = "go-tokenizer", + embed = [":go-tokenizer_lib"], + visibility = ["//visibility:public"], +) diff --git a/go/extractor/dbscheme/BUILD.bazel b/go/extractor/dbscheme/BUILD.bazel new file mode 100644 index 00000000000..efaf51684d7 --- /dev/null +++ b/go/extractor/dbscheme/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "dbscheme", + srcs = [ + "dbscheme.go", + "tables.go", + ], + importpath = "github.com/github/codeql-go/extractor/dbscheme", + visibility = ["//visibility:public"], + deps = [ + "//go/extractor/trap", + "//go/extractor/vendor/golang.org/x/tools/go/packages", + ], +) diff --git a/go/extractor/diagnostics/BUILD.bazel b/go/extractor/diagnostics/BUILD.bazel new file mode 100644 index 00000000000..8b218dc1317 --- /dev/null +++ b/go/extractor/diagnostics/BUILD.bazel @@ -0,0 +1,8 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "diagnostics", + srcs = ["diagnostics.go"], + importpath = "github.com/github/codeql-go/extractor/diagnostics", + visibility = ["//visibility:public"], +) diff --git a/go/extractor/project/BUILD.bazel b/go/extractor/project/BUILD.bazel new file mode 100644 index 00000000000..dd49b3b320f --- /dev/null +++ b/go/extractor/project/BUILD.bazel @@ -0,0 +1,21 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "project", + srcs = ["project.go"], + importpath = "github.com/github/codeql-go/extractor/project", + visibility = ["//visibility:public"], + deps = [ + "//go/extractor/diagnostics", + "//go/extractor/toolchain", + "//go/extractor/util", + "//go/extractor/vendor/golang.org/x/mod/modfile", + "//go/extractor/vendor/golang.org/x/mod/semver", + ], +) + +go_test( + name = "project_test", + srcs = ["project_test.go"], + embed = [":project"], +) diff --git a/go/extractor/srcarchive/BUILD.bazel b/go/extractor/srcarchive/BUILD.bazel new file mode 100644 index 00000000000..90664c90190 --- /dev/null +++ b/go/extractor/srcarchive/BUILD.bazel @@ -0,0 +1,17 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "srcarchive", + srcs = [ + "projectlayout.go", + "srcarchive.go", + ], + importpath = "github.com/github/codeql-go/extractor/srcarchive", + visibility = ["//visibility:public"], +) + +go_test( + name = "srcarchive_test", + srcs = ["projectlayout_test.go"], + embed = [":srcarchive"], +) diff --git a/go/extractor/toolchain/BUILD.bazel b/go/extractor/toolchain/BUILD.bazel new file mode 100644 index 00000000000..d1ce09c2632 --- /dev/null +++ b/go/extractor/toolchain/BUILD.bazel @@ -0,0 +1,18 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "toolchain", + srcs = ["toolchain.go"], + importpath = "github.com/github/codeql-go/extractor/toolchain", + visibility = ["//visibility:public"], + deps = [ + "//go/extractor/util", + "//go/extractor/vendor/golang.org/x/mod/semver", + ], +) + +go_test( + name = "toolchain_test", + srcs = ["toolchain_test.go"], + embed = [":toolchain"], +) diff --git a/go/extractor/trap/BUILD.bazel b/go/extractor/trap/BUILD.bazel new file mode 100644 index 00000000000..6c3f67b3247 --- /dev/null +++ b/go/extractor/trap/BUILD.bazel @@ -0,0 +1,23 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "trap", + srcs = [ + "labels.go", + "trapwriter.go", + "util.go", + ], + importpath = "github.com/github/codeql-go/extractor/trap", + visibility = ["//visibility:public"], + deps = [ + "//go/extractor/srcarchive", + "//go/extractor/util", + "//go/extractor/vendor/golang.org/x/tools/go/packages", + ], +) + +go_test( + name = "trap_test", + srcs = ["trapwriter_test.go"], + embed = [":trap"], +) diff --git a/go/extractor/util/BUILD.bazel b/go/extractor/util/BUILD.bazel new file mode 100644 index 00000000000..787a627b5bc --- /dev/null +++ b/go/extractor/util/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "util", + srcs = ["util.go"], + importpath = "github.com/github/codeql-go/extractor/util", + visibility = ["//visibility:public"], +) + +go_test( + name = "util_test", + srcs = ["util_test.go"], + embed = [":util"], +) diff --git a/go/extractor/vendor/golang.org/x/mod/internal/lazyregexp/BUILD.bazel b/go/extractor/vendor/golang.org/x/mod/internal/lazyregexp/BUILD.bazel new file mode 100644 index 00000000000..33c06c34bec --- /dev/null +++ b/go/extractor/vendor/golang.org/x/mod/internal/lazyregexp/BUILD.bazel @@ -0,0 +1,9 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "lazyregexp", + srcs = ["lazyre.go"], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/mod/internal/lazyregexp", + importpath = "golang.org/x/mod/internal/lazyregexp", + visibility = ["//go/extractor/vendor/golang.org/x/mod:__subpackages__"], +) diff --git a/go/extractor/vendor/golang.org/x/mod/modfile/BUILD.bazel b/go/extractor/vendor/golang.org/x/mod/modfile/BUILD.bazel new file mode 100644 index 00000000000..7b5a78d785c --- /dev/null +++ b/go/extractor/vendor/golang.org/x/mod/modfile/BUILD.bazel @@ -0,0 +1,19 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "modfile", + srcs = [ + "print.go", + "read.go", + "rule.go", + "work.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/mod/modfile", + importpath = "golang.org/x/mod/modfile", + visibility = ["//visibility:public"], + deps = [ + "//go/extractor/vendor/golang.org/x/mod/internal/lazyregexp", + "//go/extractor/vendor/golang.org/x/mod/module", + "//go/extractor/vendor/golang.org/x/mod/semver", + ], +) diff --git a/go/extractor/vendor/golang.org/x/mod/module/BUILD.bazel b/go/extractor/vendor/golang.org/x/mod/module/BUILD.bazel new file mode 100644 index 00000000000..46ff4dbb7d1 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/mod/module/BUILD.bazel @@ -0,0 +1,16 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "module", + srcs = [ + "module.go", + "pseudo.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/mod/module", + importpath = "golang.org/x/mod/module", + visibility = ["//visibility:public"], + deps = [ + "//go/extractor/vendor/golang.org/x/mod/internal/lazyregexp", + "//go/extractor/vendor/golang.org/x/mod/semver", + ], +) diff --git a/go/extractor/vendor/golang.org/x/mod/semver/BUILD.bazel b/go/extractor/vendor/golang.org/x/mod/semver/BUILD.bazel new file mode 100644 index 00000000000..5b54efe1981 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/mod/semver/BUILD.bazel @@ -0,0 +1,9 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "semver", + srcs = ["semver.go"], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/mod/semver", + importpath = "golang.org/x/mod/semver", + visibility = ["//visibility:public"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/go/gcexportdata/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/go/gcexportdata/BUILD.bazel new file mode 100644 index 00000000000..57f503f1cb2 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/go/gcexportdata/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "gcexportdata", + srcs = [ + "gcexportdata.go", + "importer.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/go/gcexportdata", + importpath = "golang.org/x/tools/go/gcexportdata", + visibility = ["//visibility:public"], + deps = ["//go/extractor/vendor/golang.org/x/tools/internal/gcimporter"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/go/internal/packagesdriver/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/go/internal/packagesdriver/BUILD.bazel new file mode 100644 index 00000000000..962442de1e0 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/go/internal/packagesdriver/BUILD.bazel @@ -0,0 +1,10 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "packagesdriver", + srcs = ["sizes.go"], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/go/internal/packagesdriver", + importpath = "golang.org/x/tools/go/internal/packagesdriver", + visibility = ["//go/extractor/vendor/golang.org/x/tools/go:__subpackages__"], + deps = ["//go/extractor/vendor/golang.org/x/tools/internal/gocommand"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/go/packages/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/go/packages/BUILD.bazel new file mode 100644 index 00000000000..1cad8baca98 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/go/packages/BUILD.bazel @@ -0,0 +1,25 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "packages", + srcs = [ + "doc.go", + "external.go", + "golist.go", + "golist_overlay.go", + "loadmode_string.go", + "packages.go", + "visit.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/go/packages", + importpath = "golang.org/x/tools/go/packages", + visibility = ["//visibility:public"], + deps = [ + "//go/extractor/vendor/golang.org/x/tools/go/gcexportdata", + "//go/extractor/vendor/golang.org/x/tools/go/internal/packagesdriver", + "//go/extractor/vendor/golang.org/x/tools/internal/gocommand", + "//go/extractor/vendor/golang.org/x/tools/internal/packagesinternal", + "//go/extractor/vendor/golang.org/x/tools/internal/typesinternal", + "//go/extractor/vendor/golang.org/x/tools/internal/versions", + ], +) diff --git a/go/extractor/vendor/golang.org/x/tools/go/types/objectpath/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/go/types/objectpath/BUILD.bazel new file mode 100644 index 00000000000..1029221f434 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/go/types/objectpath/BUILD.bazel @@ -0,0 +1,10 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "objectpath", + srcs = ["objectpath.go"], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/go/types/objectpath", + importpath = "golang.org/x/tools/go/types/objectpath", + visibility = ["//visibility:public"], + deps = ["//go/extractor/vendor/golang.org/x/tools/internal/typeparams"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/event/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/event/BUILD.bazel new file mode 100644 index 00000000000..b882fef0d8c --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/event/BUILD.bazel @@ -0,0 +1,17 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "event", + srcs = [ + "doc.go", + "event.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/event", + importpath = "golang.org/x/tools/internal/event", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], + deps = [ + "//go/extractor/vendor/golang.org/x/tools/internal/event/core", + "//go/extractor/vendor/golang.org/x/tools/internal/event/keys", + "//go/extractor/vendor/golang.org/x/tools/internal/event/label", + ], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/event/core/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/event/core/BUILD.bazel new file mode 100644 index 00000000000..36bd68eed9f --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/event/core/BUILD.bazel @@ -0,0 +1,17 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "core", + srcs = [ + "event.go", + "export.go", + "fast.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/event/core", + importpath = "golang.org/x/tools/internal/event/core", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], + deps = [ + "//go/extractor/vendor/golang.org/x/tools/internal/event/keys", + "//go/extractor/vendor/golang.org/x/tools/internal/event/label", + ], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/event/keys/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/event/keys/BUILD.bazel new file mode 100644 index 00000000000..f1674735052 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/event/keys/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "keys", + srcs = [ + "keys.go", + "standard.go", + "util.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/event/keys", + importpath = "golang.org/x/tools/internal/event/keys", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], + deps = ["//go/extractor/vendor/golang.org/x/tools/internal/event/label"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/event/label/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/event/label/BUILD.bazel new file mode 100644 index 00000000000..2329754d6cf --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/event/label/BUILD.bazel @@ -0,0 +1,9 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "label", + srcs = ["label.go"], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/event/label", + importpath = "golang.org/x/tools/internal/event/label", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/event/tag/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/event/tag/BUILD.bazel new file mode 100644 index 00000000000..276dc5f4489 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/event/tag/BUILD.bazel @@ -0,0 +1,10 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "tag", + srcs = ["tag.go"], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/event/tag", + importpath = "golang.org/x/tools/internal/event/tag", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], + deps = ["//go/extractor/vendor/golang.org/x/tools/internal/event/keys"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/gcimporter/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/gcimporter/BUILD.bazel new file mode 100644 index 00000000000..1879fe0dadf --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/gcimporter/BUILD.bazel @@ -0,0 +1,27 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "gcimporter", + srcs = [ + "bimport.go", + "exportdata.go", + "gcimporter.go", + "iexport.go", + "iimport.go", + "newInterface10.go", + "newInterface11.go", + "support_go117.go", + "support_go118.go", + "unified_no.go", + "ureader_no.go", + "ureader_yes.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/gcimporter", + importpath = "golang.org/x/tools/internal/gcimporter", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], + deps = [ + "//go/extractor/vendor/golang.org/x/tools/go/types/objectpath", + "//go/extractor/vendor/golang.org/x/tools/internal/pkgbits", + "//go/extractor/vendor/golang.org/x/tools/internal/tokeninternal", + ], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/gocommand/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/gocommand/BUILD.bazel new file mode 100644 index 00000000000..58f7091b49c --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/gocommand/BUILD.bazel @@ -0,0 +1,20 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "gocommand", + srcs = [ + "invoke.go", + "vendor.go", + "version.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/gocommand", + importpath = "golang.org/x/tools/internal/gocommand", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], + deps = [ + "//go/extractor/vendor/golang.org/x/mod/semver", + "//go/extractor/vendor/golang.org/x/tools/internal/event", + "//go/extractor/vendor/golang.org/x/tools/internal/event/keys", + "//go/extractor/vendor/golang.org/x/tools/internal/event/label", + "//go/extractor/vendor/golang.org/x/tools/internal/event/tag", + ], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/packagesinternal/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/packagesinternal/BUILD.bazel new file mode 100644 index 00000000000..3ed918c9489 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/packagesinternal/BUILD.bazel @@ -0,0 +1,9 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "packagesinternal", + srcs = ["packages.go"], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/packagesinternal", + importpath = "golang.org/x/tools/internal/packagesinternal", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/pkgbits/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/pkgbits/BUILD.bazel new file mode 100644 index 00000000000..820e8f04c9b --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/pkgbits/BUILD.bazel @@ -0,0 +1,21 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "pkgbits", + srcs = [ + "codes.go", + "decoder.go", + "doc.go", + "encoder.go", + "flags.go", + "frames_go1.go", + "frames_go17.go", + "reloc.go", + "support.go", + "sync.go", + "syncmarker_string.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/pkgbits", + importpath = "golang.org/x/tools/internal/pkgbits", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/tokeninternal/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/tokeninternal/BUILD.bazel new file mode 100644 index 00000000000..ff66085dbdb --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/tokeninternal/BUILD.bazel @@ -0,0 +1,9 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "tokeninternal", + srcs = ["tokeninternal.go"], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/tokeninternal", + importpath = "golang.org/x/tools/internal/tokeninternal", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/typeparams/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/typeparams/BUILD.bazel new file mode 100644 index 00000000000..266816d9ea2 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/typeparams/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "typeparams", + srcs = [ + "common.go", + "coretype.go", + "normalize.go", + "termlist.go", + "typeterm.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/typeparams", + importpath = "golang.org/x/tools/internal/typeparams", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/typesinternal/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/typesinternal/BUILD.bazel new file mode 100644 index 00000000000..c03b8a36ef7 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/typesinternal/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "typesinternal", + srcs = [ + "errorcode.go", + "errorcode_string.go", + "types.go", + "types_118.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/typesinternal", + importpath = "golang.org/x/tools/internal/typesinternal", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/versions/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/versions/BUILD.bazel new file mode 100644 index 00000000000..6d7ffd0ac68 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/versions/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "versions", + srcs = [ + "gover.go", + "types.go", + "types_go121.go", + "types_go122.go", + "versions.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/versions", + importpath = "golang.org/x/tools/internal/versions", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], +) diff --git a/go/gazelle/BUILD.bazel b/go/gazelle/BUILD.bazel new file mode 100644 index 00000000000..7c9b34edb3a --- /dev/null +++ b/go/gazelle/BUILD.bazel @@ -0,0 +1,8 @@ +load("@gazelle//:def.bzl", "gazelle") + +gazelle( + name = "gazelle", + extra_args = [ + "go/extractor", + ], +) diff --git a/go/rules.bzl b/go/rules.bzl new file mode 100644 index 00000000000..e69de29bb2d From 4ca8faa9c944faa1005046c2c5cef3f6d68131b3 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 25 Apr 2024 09:14:59 +0200 Subject: [PATCH 02/41] Go: introduce universal binaries on macOS --- MODULE.bazel | 3 ++- go/BUILD.bazel | 0 go/extractor/BUILD.bazel | 1 + go/extractor/cli/go-autobuilder/BUILD.bazel | 5 ++-- go/extractor/cli/go-bootstrap/BUILD.bazel | 5 ++-- go/extractor/cli/go-build-runner/BUILD.bazel | 5 ++-- go/extractor/cli/go-extractor/BUILD.bazel | 5 ++-- go/extractor/cli/go-gen-dbscheme/BUILD.bazel | 5 ++-- go/extractor/cli/go-tokenizer/BUILD.bazel | 5 ++-- go/rules.bzl | 5 ++++ misc/bazel/universal_binary.bzl | 24 ++++++++++++++++++++ 11 files changed, 50 insertions(+), 13 deletions(-) create mode 100644 go/BUILD.bazel create mode 100644 misc/bazel/universal_binary.bzl diff --git a/MODULE.bazel b/MODULE.bazel index 2dc9dd0a7bb..7a85ab51a96 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -13,7 +13,8 @@ local_path_override( # see https://registry.bazel.build/ for a list of available packages -bazel_dep(name = "platforms", version = "0.0.8") +bazel_dep(name = "apple_support", version = "1.15.1") +bazel_dep(name = "platforms", version = "0.0.9") bazel_dep(name = "rules_go", version = "0.47.0") bazel_dep(name = "rules_pkg", version = "0.10.1") bazel_dep(name = "rules_nodejs", version = "6.0.3") diff --git a/go/BUILD.bazel b/go/BUILD.bazel new file mode 100644 index 00000000000..e69de29bb2d diff --git a/go/extractor/BUILD.bazel b/go/extractor/BUILD.bazel index ea99340e7d7..040b4227314 100644 --- a/go/extractor/BUILD.bazel +++ b/go/extractor/BUILD.bazel @@ -1,6 +1,7 @@ load("@rules_go//go:def.bzl", "go_library") # gazelle:prefix github.com/github/codeql-go/extractor +# gazelle:map_kind go_binary codeql_go_binary //go:rules.bzl go_library( name = "extractor", diff --git a/go/extractor/cli/go-autobuilder/BUILD.bazel b/go/extractor/cli/go-autobuilder/BUILD.bazel index b53227b9f1b..bf1235b33aa 100644 --- a/go/extractor/cli/go-autobuilder/BUILD.bazel +++ b/go/extractor/cli/go-autobuilder/BUILD.bazel @@ -1,4 +1,5 @@ -load("@rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_go//go:def.bzl", "go_library") +load("//go:rules.bzl", "codeql_go_binary") go_library( name = "go-autobuilder_lib", @@ -15,7 +16,7 @@ go_library( ], ) -go_binary( +codeql_go_binary( name = "go-autobuilder", embed = [":go-autobuilder_lib"], visibility = ["//visibility:public"], diff --git a/go/extractor/cli/go-bootstrap/BUILD.bazel b/go/extractor/cli/go-bootstrap/BUILD.bazel index 15255227f9a..7bdd6d6e70f 100644 --- a/go/extractor/cli/go-bootstrap/BUILD.bazel +++ b/go/extractor/cli/go-bootstrap/BUILD.bazel @@ -1,4 +1,5 @@ -load("@rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_go//go:def.bzl", "go_library") +load("//go:rules.bzl", "codeql_go_binary") go_library( name = "go-bootstrap_lib", @@ -7,7 +8,7 @@ go_library( visibility = ["//visibility:private"], ) -go_binary( +codeql_go_binary( name = "go-bootstrap", embed = [":go-bootstrap_lib"], visibility = ["//visibility:public"], diff --git a/go/extractor/cli/go-build-runner/BUILD.bazel b/go/extractor/cli/go-build-runner/BUILD.bazel index fc407c3e172..15557d751cf 100644 --- a/go/extractor/cli/go-build-runner/BUILD.bazel +++ b/go/extractor/cli/go-build-runner/BUILD.bazel @@ -1,4 +1,5 @@ -load("@rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_go//go:def.bzl", "go_library") +load("//go:rules.bzl", "codeql_go_binary") go_library( name = "go-build-runner_lib", @@ -11,7 +12,7 @@ go_library( ], ) -go_binary( +codeql_go_binary( name = "go-build-runner", embed = [":go-build-runner_lib"], visibility = ["//visibility:public"], diff --git a/go/extractor/cli/go-extractor/BUILD.bazel b/go/extractor/cli/go-extractor/BUILD.bazel index 9419c6f8ba9..be426331868 100644 --- a/go/extractor/cli/go-extractor/BUILD.bazel +++ b/go/extractor/cli/go-extractor/BUILD.bazel @@ -1,4 +1,5 @@ -load("@rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_go//go:def.bzl", "go_library") +load("//go:rules.bzl", "codeql_go_binary") go_library( name = "go-extractor_lib", @@ -11,7 +12,7 @@ go_library( ], ) -go_binary( +codeql_go_binary( name = "go-extractor", embed = [":go-extractor_lib"], visibility = ["//visibility:public"], diff --git a/go/extractor/cli/go-gen-dbscheme/BUILD.bazel b/go/extractor/cli/go-gen-dbscheme/BUILD.bazel index 1fbd75998c2..06c0d0f61f1 100644 --- a/go/extractor/cli/go-gen-dbscheme/BUILD.bazel +++ b/go/extractor/cli/go-gen-dbscheme/BUILD.bazel @@ -1,4 +1,5 @@ -load("@rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_go//go:def.bzl", "go_library") +load("//go:rules.bzl", "codeql_go_binary") go_library( name = "go-gen-dbscheme_lib", @@ -8,7 +9,7 @@ go_library( deps = ["//go/extractor/dbscheme"], ) -go_binary( +codeql_go_binary( name = "go-gen-dbscheme", embed = [":go-gen-dbscheme_lib"], visibility = ["//visibility:public"], diff --git a/go/extractor/cli/go-tokenizer/BUILD.bazel b/go/extractor/cli/go-tokenizer/BUILD.bazel index f55baecc69f..3fc5b464c7c 100644 --- a/go/extractor/cli/go-tokenizer/BUILD.bazel +++ b/go/extractor/cli/go-tokenizer/BUILD.bazel @@ -1,4 +1,5 @@ -load("@rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_go//go:def.bzl", "go_library") +load("//go:rules.bzl", "codeql_go_binary") go_library( name = "go-tokenizer_lib", @@ -7,7 +8,7 @@ go_library( visibility = ["//visibility:private"], ) -go_binary( +codeql_go_binary( name = "go-tokenizer", embed = [":go-tokenizer_lib"], visibility = ["//visibility:public"], diff --git a/go/rules.bzl b/go/rules.bzl index e69de29bb2d..4ef798001d2 100644 --- a/go/rules.bzl +++ b/go/rules.bzl @@ -0,0 +1,5 @@ +load("@rules_go//go:def.bzl", "go_binary") +load("//misc/bazel:universal_binary.bzl", "wrap_as_universal_binary") + +def codeql_go_binary(**kwargs): + wrap_as_universal_binary(go_binary, **kwargs) diff --git a/misc/bazel/universal_binary.bzl b/misc/bazel/universal_binary.bzl new file mode 100644 index 00000000000..85881356d0e --- /dev/null +++ b/misc/bazel/universal_binary.bzl @@ -0,0 +1,24 @@ +load("@apple_support//rules:universal_binary.bzl", _universal_binary = "universal_binary") + +def wrap_as_universal_binary(rule, *, name, visibility = None, **kwargs): + internal_name = "internal/%s" % name + universal_name = "universal/%s" % name + rule( + name = internal_name, + visibility = ["//visibility:private"], + **kwargs + ) + _universal_binary( + name = universal_name, + target_compatible_with = ["@platforms//os:macos"], + binary = internal_name, + visibility = ["//visibility:private"], + ) + native.alias( + name = name, + actual = select({ + "@platforms//os:macos": universal_name, + "//conditions:default": internal_name, + }), + visibility = visibility, + ) From 7d9a68bf173511af23c94bbd04560c42a123dc09 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 26 Apr 2024 11:19:22 +0200 Subject: [PATCH 03/41] Go: wrap gazelle to regenerate from scratch and add header --- go/extractor/autobuilder/BUILD.bazel | 2 ++ go/extractor/cli/go-autobuilder/BUILD.bazel | 2 ++ go/extractor/cli/go-bootstrap/BUILD.bazel | 2 ++ go/extractor/cli/go-build-runner/BUILD.bazel | 2 ++ go/extractor/cli/go-extractor/BUILD.bazel | 2 ++ go/extractor/cli/go-gen-dbscheme/BUILD.bazel | 2 ++ go/extractor/cli/go-tokenizer/BUILD.bazel | 2 ++ go/extractor/dbscheme/BUILD.bazel | 2 ++ go/extractor/diagnostics/BUILD.bazel | 2 ++ go/extractor/project/BUILD.bazel | 2 ++ go/extractor/srcarchive/BUILD.bazel | 2 ++ go/extractor/toolchain/BUILD.bazel | 2 ++ go/extractor/trap/BUILD.bazel | 2 ++ go/extractor/util/BUILD.bazel | 2 ++ .../x/mod/internal/lazyregexp/BUILD.bazel | 2 ++ .../vendor/golang.org/x/mod/modfile/BUILD.bazel | 2 ++ .../vendor/golang.org/x/mod/module/BUILD.bazel | 2 ++ .../vendor/golang.org/x/mod/semver/BUILD.bazel | 2 ++ .../x/tools/go/gcexportdata/BUILD.bazel | 2 ++ .../tools/go/internal/packagesdriver/BUILD.bazel | 2 ++ .../golang.org/x/tools/go/packages/BUILD.bazel | 2 ++ .../x/tools/go/types/objectpath/BUILD.bazel | 2 ++ .../x/tools/internal/event/BUILD.bazel | 2 ++ .../x/tools/internal/event/core/BUILD.bazel | 2 ++ .../x/tools/internal/event/keys/BUILD.bazel | 2 ++ .../x/tools/internal/event/label/BUILD.bazel | 2 ++ .../x/tools/internal/event/tag/BUILD.bazel | 2 ++ .../x/tools/internal/gcimporter/BUILD.bazel | 2 ++ .../x/tools/internal/gocommand/BUILD.bazel | 2 ++ .../tools/internal/packagesinternal/BUILD.bazel | 2 ++ .../x/tools/internal/pkgbits/BUILD.bazel | 2 ++ .../x/tools/internal/tokeninternal/BUILD.bazel | 2 ++ .../x/tools/internal/typeparams/BUILD.bazel | 2 ++ .../x/tools/internal/typesinternal/BUILD.bazel | 2 ++ .../x/tools/internal/versions/BUILD.bazel | 2 ++ go/gazelle/BUILD.bazel | 13 +++++++++---- go/gazelle/gazelle.py | 16 ++++++++++++++++ 37 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 go/gazelle/gazelle.py diff --git a/go/extractor/autobuilder/BUILD.bazel b/go/extractor/autobuilder/BUILD.bazel index b81b15816aa..e40dc3a0321 100644 --- a/go/extractor/autobuilder/BUILD.bazel +++ b/go/extractor/autobuilder/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( diff --git a/go/extractor/cli/go-autobuilder/BUILD.bazel b/go/extractor/cli/go-autobuilder/BUILD.bazel index bf1235b33aa..7abf4600d94 100644 --- a/go/extractor/cli/go-autobuilder/BUILD.bazel +++ b/go/extractor/cli/go-autobuilder/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") load("//go:rules.bzl", "codeql_go_binary") diff --git a/go/extractor/cli/go-bootstrap/BUILD.bazel b/go/extractor/cli/go-bootstrap/BUILD.bazel index 7bdd6d6e70f..86b08dfa121 100644 --- a/go/extractor/cli/go-bootstrap/BUILD.bazel +++ b/go/extractor/cli/go-bootstrap/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") load("//go:rules.bzl", "codeql_go_binary") diff --git a/go/extractor/cli/go-build-runner/BUILD.bazel b/go/extractor/cli/go-build-runner/BUILD.bazel index 15557d751cf..e91c4530610 100644 --- a/go/extractor/cli/go-build-runner/BUILD.bazel +++ b/go/extractor/cli/go-build-runner/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") load("//go:rules.bzl", "codeql_go_binary") diff --git a/go/extractor/cli/go-extractor/BUILD.bazel b/go/extractor/cli/go-extractor/BUILD.bazel index be426331868..769e4a7b09b 100644 --- a/go/extractor/cli/go-extractor/BUILD.bazel +++ b/go/extractor/cli/go-extractor/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") load("//go:rules.bzl", "codeql_go_binary") diff --git a/go/extractor/cli/go-gen-dbscheme/BUILD.bazel b/go/extractor/cli/go-gen-dbscheme/BUILD.bazel index 06c0d0f61f1..9de6d2198f8 100644 --- a/go/extractor/cli/go-gen-dbscheme/BUILD.bazel +++ b/go/extractor/cli/go-gen-dbscheme/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") load("//go:rules.bzl", "codeql_go_binary") diff --git a/go/extractor/cli/go-tokenizer/BUILD.bazel b/go/extractor/cli/go-tokenizer/BUILD.bazel index 3fc5b464c7c..8ce0c76be26 100644 --- a/go/extractor/cli/go-tokenizer/BUILD.bazel +++ b/go/extractor/cli/go-tokenizer/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") load("//go:rules.bzl", "codeql_go_binary") diff --git a/go/extractor/dbscheme/BUILD.bazel b/go/extractor/dbscheme/BUILD.bazel index efaf51684d7..496a5ccdf49 100644 --- a/go/extractor/dbscheme/BUILD.bazel +++ b/go/extractor/dbscheme/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/diagnostics/BUILD.bazel b/go/extractor/diagnostics/BUILD.bazel index 8b218dc1317..9b0c148db32 100644 --- a/go/extractor/diagnostics/BUILD.bazel +++ b/go/extractor/diagnostics/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/project/BUILD.bazel b/go/extractor/project/BUILD.bazel index dd49b3b320f..0048adf9164 100644 --- a/go/extractor/project/BUILD.bazel +++ b/go/extractor/project/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( diff --git a/go/extractor/srcarchive/BUILD.bazel b/go/extractor/srcarchive/BUILD.bazel index 90664c90190..e72e2e7ca08 100644 --- a/go/extractor/srcarchive/BUILD.bazel +++ b/go/extractor/srcarchive/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( diff --git a/go/extractor/toolchain/BUILD.bazel b/go/extractor/toolchain/BUILD.bazel index d1ce09c2632..fde8d327e9e 100644 --- a/go/extractor/toolchain/BUILD.bazel +++ b/go/extractor/toolchain/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( diff --git a/go/extractor/trap/BUILD.bazel b/go/extractor/trap/BUILD.bazel index 6c3f67b3247..6cc7c4983b2 100644 --- a/go/extractor/trap/BUILD.bazel +++ b/go/extractor/trap/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( diff --git a/go/extractor/util/BUILD.bazel b/go/extractor/util/BUILD.bazel index 787a627b5bc..8b8869cac52 100644 --- a/go/extractor/util/BUILD.bazel +++ b/go/extractor/util/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( diff --git a/go/extractor/vendor/golang.org/x/mod/internal/lazyregexp/BUILD.bazel b/go/extractor/vendor/golang.org/x/mod/internal/lazyregexp/BUILD.bazel index 33c06c34bec..deb5dc2b019 100644 --- a/go/extractor/vendor/golang.org/x/mod/internal/lazyregexp/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/mod/internal/lazyregexp/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/mod/modfile/BUILD.bazel b/go/extractor/vendor/golang.org/x/mod/modfile/BUILD.bazel index 7b5a78d785c..097bacb107c 100644 --- a/go/extractor/vendor/golang.org/x/mod/modfile/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/mod/modfile/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/mod/module/BUILD.bazel b/go/extractor/vendor/golang.org/x/mod/module/BUILD.bazel index 46ff4dbb7d1..3bf5ae9997d 100644 --- a/go/extractor/vendor/golang.org/x/mod/module/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/mod/module/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/mod/semver/BUILD.bazel b/go/extractor/vendor/golang.org/x/mod/semver/BUILD.bazel index 5b54efe1981..760be56c9e0 100644 --- a/go/extractor/vendor/golang.org/x/mod/semver/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/mod/semver/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/go/gcexportdata/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/go/gcexportdata/BUILD.bazel index 57f503f1cb2..5d68c2fe989 100644 --- a/go/extractor/vendor/golang.org/x/tools/go/gcexportdata/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/go/gcexportdata/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/go/internal/packagesdriver/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/go/internal/packagesdriver/BUILD.bazel index 962442de1e0..2ef27e2c88a 100644 --- a/go/extractor/vendor/golang.org/x/tools/go/internal/packagesdriver/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/go/internal/packagesdriver/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/go/packages/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/go/packages/BUILD.bazel index 1cad8baca98..03d3e3b0158 100644 --- a/go/extractor/vendor/golang.org/x/tools/go/packages/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/go/packages/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/go/types/objectpath/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/go/types/objectpath/BUILD.bazel index 1029221f434..374c5c601bc 100644 --- a/go/extractor/vendor/golang.org/x/tools/go/types/objectpath/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/go/types/objectpath/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/internal/event/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/event/BUILD.bazel index b882fef0d8c..200e436fcd4 100644 --- a/go/extractor/vendor/golang.org/x/tools/internal/event/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/internal/event/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/internal/event/core/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/event/core/BUILD.bazel index 36bd68eed9f..a16713f536c 100644 --- a/go/extractor/vendor/golang.org/x/tools/internal/event/core/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/internal/event/core/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/internal/event/keys/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/event/keys/BUILD.bazel index f1674735052..1feefdf1a83 100644 --- a/go/extractor/vendor/golang.org/x/tools/internal/event/keys/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/internal/event/keys/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/internal/event/label/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/event/label/BUILD.bazel index 2329754d6cf..a4430ba0a17 100644 --- a/go/extractor/vendor/golang.org/x/tools/internal/event/label/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/internal/event/label/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/internal/event/tag/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/event/tag/BUILD.bazel index 276dc5f4489..d2c87f41a8a 100644 --- a/go/extractor/vendor/golang.org/x/tools/internal/event/tag/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/internal/event/tag/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/internal/gcimporter/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/gcimporter/BUILD.bazel index 1879fe0dadf..56da3b0130e 100644 --- a/go/extractor/vendor/golang.org/x/tools/internal/gcimporter/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/internal/gcimporter/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/internal/gocommand/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/gocommand/BUILD.bazel index 58f7091b49c..7e64f94b95c 100644 --- a/go/extractor/vendor/golang.org/x/tools/internal/gocommand/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/internal/gocommand/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/internal/packagesinternal/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/packagesinternal/BUILD.bazel index 3ed918c9489..2d2b7dc5b33 100644 --- a/go/extractor/vendor/golang.org/x/tools/internal/packagesinternal/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/internal/packagesinternal/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/internal/pkgbits/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/pkgbits/BUILD.bazel index 820e8f04c9b..cce32747051 100644 --- a/go/extractor/vendor/golang.org/x/tools/internal/pkgbits/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/internal/pkgbits/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/internal/tokeninternal/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/tokeninternal/BUILD.bazel index ff66085dbdb..c0f6cc8fb13 100644 --- a/go/extractor/vendor/golang.org/x/tools/internal/tokeninternal/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/internal/tokeninternal/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/internal/typeparams/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/typeparams/BUILD.bazel index 266816d9ea2..9c2dc20b6c6 100644 --- a/go/extractor/vendor/golang.org/x/tools/internal/typeparams/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/internal/typeparams/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/internal/typesinternal/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/typesinternal/BUILD.bazel index c03b8a36ef7..653752ab715 100644 --- a/go/extractor/vendor/golang.org/x/tools/internal/typesinternal/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/internal/typesinternal/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/internal/versions/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/versions/BUILD.bazel index 6d7ffd0ac68..85d428debf5 100644 --- a/go/extractor/vendor/golang.org/x/tools/internal/versions/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/internal/versions/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/gazelle/BUILD.bazel b/go/gazelle/BUILD.bazel index 7c9b34edb3a..3d81b50590a 100644 --- a/go/gazelle/BUILD.bazel +++ b/go/gazelle/BUILD.bazel @@ -1,8 +1,13 @@ load("@gazelle//:def.bzl", "gazelle") gazelle( - name = "gazelle", - extra_args = [ - "go/extractor", - ], + name = "_gazelle", +) + +py_binary( + name = "gazelle", + srcs = ["gazelle.py"], + args = ["$(rlocationpath :_gazelle)"], + data = [":_gazelle"], + deps = ["@rules_python//python/runfiles"], ) diff --git a/go/gazelle/gazelle.py b/go/gazelle/gazelle.py new file mode 100644 index 00000000000..200f3c3ed6b --- /dev/null +++ b/go/gazelle/gazelle.py @@ -0,0 +1,16 @@ +import sys +import pathlib +import subprocess +from python.runfiles import runfiles + +this = pathlib.Path(__file__).resolve() +go_extractor_dir = this.parents[1] / "extractor" +gazelle = runfiles.Create().Rlocation(sys.argv[1]) +for build_file in go_extractor_dir.glob("*/**/BUILD.bazel"): + build_file.unlink() + +subprocess.check_call([gazelle, "go/extractor"]) + +for build_file in go_extractor_dir.glob("*/**/BUILD.bazel"): + contents = build_file.read_text() + build_file.write_text(f"# generated running `bazel run //go/gazelle`, do not edit\n\n{contents}") From 3ad9c026a5c59b8a23d5541a1251fc611dd89988 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 26 Apr 2024 11:20:47 +0200 Subject: [PATCH 04/41] Go: remove `go_sdk.host` It's not required, and it can't work from the internal repository. --- MODULE.bazel | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 7a85ab51a96..16697403dcc 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -23,8 +23,7 @@ bazel_dep(name = "bazel_skylib", version = "1.5.0") bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "absl") bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "json") bazel_dep(name = "fmt", version = "10.0.0") - -bazel_dep(name = "gazelle", version = "0.36.0", dev_dependency = True) +bazel_dep(name = "gazelle", version = "0.36.0") pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") pip.parse( @@ -56,7 +55,6 @@ use_repo(node, "nodejs", "nodejs_toolchains") go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk") go_sdk.download(version = "1.22.2") -go_sdk.host() register_toolchains( "@nodejs_toolchains//:all", From 925a2cca7e6a5c8231b37f67df3de469ddf50925 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 26 Apr 2024 12:01:23 +0200 Subject: [PATCH 05/41] Go: create whole extractor pack with bazel --- go/BUILD.bazel | 59 +++++++++++++++++++++++++++++++++++++ go/codeql-tools/BUILD.bazel | 28 ++++++++++++++++++ go/create_extractor_pack.py | 16 ++++++++++ go/downgrades/BUILD.bazel | 12 ++++++++ go/extractor/BUILD.bazel | 31 +++++++++++++++++++ 5 files changed, 146 insertions(+) create mode 100644 go/codeql-tools/BUILD.bazel create mode 100644 go/create_extractor_pack.py create mode 100644 go/downgrades/BUILD.bazel diff --git a/go/BUILD.bazel b/go/BUILD.bazel index e69de29bb2d..5736912310a 100644 --- a/go/BUILD.bazel +++ b/go/BUILD.bazel @@ -0,0 +1,59 @@ +load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup", "pkg_files") +load("@rules_pkg//pkg:install.bzl", "pkg_install") +load("//:defs.bzl", "codeql_platform") + +pkg_files( + name = "resources", + srcs = [ + "LICENSE", + "codeql-extractor.yml", + "ql/lib/go.dbscheme", + "ql/lib/go.dbscheme.stats", + ], +) + +pkg_filegroup( + name = "extractor-pack-generic", + srcs = [ + ":resources", + "//go/codeql-tools", + "//go/downgrades", + "//go/extractor:tokenizer", + ], + visibility = ["//visibility:public"], +) + +pkg_files( + name = "extractor-pack-arch", + srcs = [ + "//go/extractor/cli/go-autobuilder", + "//go/extractor/cli/go-bootstrap", + "//go/extractor/cli/go-build-runner", + "//go/extractor/cli/go-extractor", + "//go/extractor/cli/go-gen-dbscheme", + "//go/extractor/cli/go-tokenizer", + ], + prefix = "tools/" + codeql_platform, + visibility = ["//visibility:public"], +) + +pkg_filegroup( + name = "extractor-pack", + srcs = [ + ":extractor-pack-arch", + ":extractor-pack-generic", + ], + visibility = ["//visibility:public"], +) + +pkg_install( + name = "_create_extractor_pack", + srcs = [":extractor-pack"], +) + +py_binary( + name = "create-extractor-pack", + srcs = ["create_extractor_pack.py"], + main = "create_extractor_pack.py", + deps = [":_create_extractor_pack"], +) diff --git a/go/codeql-tools/BUILD.bazel b/go/codeql-tools/BUILD.bazel new file mode 100644 index 00000000000..4e839b4774b --- /dev/null +++ b/go/codeql-tools/BUILD.bazel @@ -0,0 +1,28 @@ +load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files") + +pkg_files( + name = "executables", + srcs = glob(["*.sh"]), + attributes = pkg_attributes(mode = "0755"), +) + +pkg_files( + name = "non-executables", + srcs = glob( + ["*"], + exclude = [ + "*.sh", + "BUILD.bazel", + ], + ), +) + +pkg_filegroup( + name = "codeql-tools", + srcs = [ + ":executables", + ":non-executables", + ], + prefix = "tools", + visibility = ["//go:__pkg__"], +) diff --git a/go/create_extractor_pack.py b/go/create_extractor_pack.py new file mode 100644 index 00000000000..08665a2d8dc --- /dev/null +++ b/go/create_extractor_pack.py @@ -0,0 +1,16 @@ +import os +import pathlib +import shutil +import sys +from go._create_extractor_pack_install_script import main + +try: + workspace_dir = pathlib.Path(os.environ['BUILD_WORKSPACE_DIRECTORY']) +except KeyError: + print("this should be run with bazel run", file=sys.stderr) + sys.exit(1) + +dest_dir = workspace_dir / 'go' / 'build' / 'codeql-extractor-go' +shutil.rmtree(dest_dir, ignore_errors=True) +os.environ['DESTDIR'] = str(dest_dir) +main(sys.argv) diff --git a/go/downgrades/BUILD.bazel b/go/downgrades/BUILD.bazel new file mode 100644 index 00000000000..68c15741a9f --- /dev/null +++ b/go/downgrades/BUILD.bazel @@ -0,0 +1,12 @@ +load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "strip_prefix") + +pkg_files( + name = "downgrades", + srcs = glob( + ["**"], + exclude = ["BUILD.bazel"], + ), + prefix = "downgrades", + strip_prefix = strip_prefix.from_pkg(), + visibility = ["//go:__pkg__"], +) diff --git a/go/extractor/BUILD.bazel b/go/extractor/BUILD.bazel index 040b4227314..cb7bc3ac8ef 100644 --- a/go/extractor/BUILD.bazel +++ b/go/extractor/BUILD.bazel @@ -1,4 +1,5 @@ load("@rules_go//go:def.bzl", "go_library") +load("@rules_pkg//pkg:mappings.bzl", "pkg_files") # gazelle:prefix github.com/github/codeql-go/extractor # gazelle:map_kind go_binary codeql_go_binary //go:rules.bzl @@ -22,3 +23,33 @@ go_library( "//go/extractor/vendor/golang.org/x/tools/go/packages", ], ) + +java_library( + name = "tokenizer-deps", + srcs = [ + "net/sourceforge/pmd/cpd/AbstractLanguage.java", + "net/sourceforge/pmd/cpd/SourceCode.java", + "net/sourceforge/pmd/cpd/TokenEntry.java", + "net/sourceforge/pmd/cpd/Tokenizer.java", + ], +) + +java_library( + name = "tokenizer-jar", + srcs = [ + "net/sourceforge/pmd/cpd/GoLanguage.java", + "opencsv/CSVParser.java", + "opencsv/CSVReader.java", + ], + deps = [":tokenizer-deps"], +) + +pkg_files( + name = "tokenizer", + srcs = [":tokenizer-jar"], + prefix = "tools", + renames = { + ":tokenizer-jar": "tokenizer.jar", + }, + visibility = ["//go:__pkg__"], +) From 19b2e56d0276518c1c6559536338b61e2451acf6 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 26 Apr 2024 13:38:10 +0200 Subject: [PATCH 06/41] Go: group BUILD and dbscheme generation target --- .pre-commit-config.yaml | 8 ++++---- go/BUILD.bazel | 28 ++++++++++++++++++++++++++++ go/gazelle/BUILD.bazel | 13 ------------- go/{gazelle/gazelle.py => gen.py} | 12 ++++++++++-- 4 files changed, 42 insertions(+), 19 deletions(-) delete mode 100644 go/gazelle/BUILD.bazel rename go/{gazelle/gazelle.py => gen.py} (56%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4e81bcc7711..899ddd71b89 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,11 +27,11 @@ repos: - repo: local hooks: - - id: gazelle - name: Check gazelle-generated BUILD files - files: go/extractor/.* + - id: go-gen + name: Check checked in generated files in go + files: go/.* language: system - entry: bazel run //go/gazelle + entry: bazel run //go:gen pass_filenames: false - id: codeql-format diff --git a/go/BUILD.bazel b/go/BUILD.bazel index 5736912310a..c5f32c5309f 100644 --- a/go/BUILD.bazel +++ b/go/BUILD.bazel @@ -1,7 +1,26 @@ load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup", "pkg_files") load("@rules_pkg//pkg:install.bzl", "pkg_install") +load("@bazel_skylib//rules:native_binary.bzl", "native_binary") +load("@gazelle//:def.bzl", "gazelle") load("//:defs.bzl", "codeql_platform") +gazelle( + name = "_gazelle", +) + +_gen_binaries = [ + ":_gazelle", + "//go/extractor/cli/go-gen-dbscheme", +] + +py_binary( + name = "gen", + srcs = ["gen.py"], + args = ["$(rlocationpath %s)" % bin for bin in _gen_binaries], + data = _gen_binaries, + deps = ["@rules_python//python/runfiles"], +) + pkg_files( name = "resources", srcs = [ @@ -57,3 +76,12 @@ py_binary( main = "create_extractor_pack.py", deps = [":_create_extractor_pack"], ) + +native_binary( + name = "gen-dbscheme", + src = "//go/extractor/cli/go-gen-dbscheme", + out = "go-gen-dbscheme", + args = [ + "$$BUILD_WORKSPACE_DIRECTORY/go/ql/lib/go.dbscheme", + ], +) diff --git a/go/gazelle/BUILD.bazel b/go/gazelle/BUILD.bazel deleted file mode 100644 index 3d81b50590a..00000000000 --- a/go/gazelle/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@gazelle//:def.bzl", "gazelle") - -gazelle( - name = "_gazelle", -) - -py_binary( - name = "gazelle", - srcs = ["gazelle.py"], - args = ["$(rlocationpath :_gazelle)"], - data = [":_gazelle"], - deps = ["@rules_python//python/runfiles"], -) diff --git a/go/gazelle/gazelle.py b/go/gen.py similarity index 56% rename from go/gazelle/gazelle.py rename to go/gen.py index 200f3c3ed6b..7b7e9d73237 100644 --- a/go/gazelle/gazelle.py +++ b/go/gen.py @@ -4,13 +4,21 @@ import subprocess from python.runfiles import runfiles this = pathlib.Path(__file__).resolve() -go_extractor_dir = this.parents[1] / "extractor" -gazelle = runfiles.Create().Rlocation(sys.argv[1]) +go_extractor_dir = this.parent / "extractor" +go_dbscheme = this.parent / "ql" / "lib" / "go.dbscheme" +r = runfiles.Create() +gazelle, go_gen_dbscheme = map(r.Rlocation, sys.argv[1:]) + +print("clearing generated BUILD files") for build_file in go_extractor_dir.glob("*/**/BUILD.bazel"): build_file.unlink() +print("running gazelle") subprocess.check_call([gazelle, "go/extractor"]) +print("adding header to generated BUILD files") for build_file in go_extractor_dir.glob("*/**/BUILD.bazel"): contents = build_file.read_text() build_file.write_text(f"# generated running `bazel run //go/gazelle`, do not edit\n\n{contents}") + +subprocess.check_call([go_gen_dbscheme, go_dbscheme]) From bfa189e2ac2c251518cb61f9665f0970b36e099d Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 26 Apr 2024 13:48:44 +0200 Subject: [PATCH 07/41] Go: use a dbscheme generated during the build in `extractor-pack` --- go/BUILD.bazel | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/go/BUILD.bazel b/go/BUILD.bazel index c5f32c5309f..d327a406f5f 100644 --- a/go/BUILD.bazel +++ b/go/BUILD.bazel @@ -21,13 +21,23 @@ py_binary( deps = ["@rules_python//python/runfiles"], ) +# this is an internal copy of the dbscheme to be used by extractor-pack +# this allows the extractor-pack target to be independent and up-to-date with respect to +# having run //go:gen to update the checked in files +genrule( + name = "dbscheme", + outs = ["go.dbscheme"], + cmd = "$(execpath //go/extractor/cli/go-gen-dbscheme) $@", + tools = ["//go/extractor/cli/go-gen-dbscheme"], +) + pkg_files( name = "resources", srcs = [ "LICENSE", "codeql-extractor.yml", - "ql/lib/go.dbscheme", "ql/lib/go.dbscheme.stats", + ":dbscheme", ], ) From 146d84bbf8c0012cb3e496a4deb7470afb61a0dd Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 26 Apr 2024 15:20:33 +0200 Subject: [PATCH 08/41] Go: rework makefile --- go/Makefile | 75 +++++------------------------------------------------ 1 file changed, 7 insertions(+), 68 deletions(-) diff --git a/go/Makefile b/go/Makefile index d0289a093a5..d9ba2c7e2d4 100644 --- a/go/Makefile +++ b/go/Makefile @@ -1,4 +1,4 @@ -all: extractor ql/lib/go.dbscheme +all: gen extractor ifeq ($(OS),Windows_NT) EXE = .exe @@ -14,17 +14,11 @@ CODEQL_PLATFORM = osx64 endif endif -CODEQL_TOOLS = $(addprefix codeql-tools/,autobuild.cmd autobuild.sh baseline-config-empty.json baseline-config-vendor.json configure-baseline.cmd configure-baseline.sh identify-environment.cmd identify-environment.sh index.cmd index.sh pre-finalize.cmd pre-finalize.sh tracing-config.lua) - EXTRACTOR_PACK_OUT = build/codeql-extractor-go -BINARIES = go-extractor go-tokenizer go-autobuilder go-build-runner go-bootstrap go-gen-dbscheme - -.PHONY: tools tools-codeql tools-codeql-full clean autoformat \ - tools-linux64 tools-osx64 tools-win64 check-formatting +.PHONY: extractor gen clean autoformat check-formatting clean: - rm -rf tools/bin tools/linux64 tools/osx64 tools/win64 tools/net tools/opencsv rm -rf $(EXTRACTOR_PACK_OUT) build/stats build/testdb autoformat: @@ -47,66 +41,11 @@ endif qhelp-to-markdown: scripts/qhelp-to-markdown.sh ql/src "$(QHELP_OUT_DIR)" -tools: tools-codeql tools/tokenizer.jar +extractor: + bazel run :create-extractor-pack -.PHONY: $(addsuffix $(EXE),$(addprefix tools/bin/,$(BINARIES))) -$(addsuffix $(EXE),$(addprefix tools/bin/,$(BINARIES))): - go build -C extractor -mod=vendor -o ../$@ ./cli/$(basename $(@F)) - -tools-codeql: tools-$(CODEQL_PLATFORM) - -tools-codeql-full: tools-linux64 tools-osx64 tools-win64 - -tools-linux64: $(addprefix tools/linux64/,$(BINARIES)) - -.PHONY: $(addprefix tools/linux64/,$(BINARIES)) -$(addprefix tools/linux64/,$(BINARIES)): - GOOS=linux GOARCH=amd64 go build -C extractor -mod=vendor -o ../$@ ./cli/$(@F) - -tools-osx64: $(addprefix tools/osx64/,$(BINARIES)) - -.PHONY: $(addprefix tools/osx64/,$(BINARIES)) -$(addprefix tools/osx64/,$(BINARIES)): - GOOS=darwin GOARCH=amd64 go build -C extractor -mod=vendor -o ../$@.amd64 ./cli/$(@F) - GOOS=darwin GOARCH=arm64 go build -C extractor -mod=vendor -o ../$@.arm64 ./cli/$(@F) - lipo -create $@.amd64 $@.arm64 -output $@ - rm $@.amd64 $@.arm64 - -tools-win64: $(addsuffix .exe,$(addprefix tools/win64/,$(BINARIES))) - -.PHONY: $(addsuffix .exe,$(addprefix tools/win64/,$(BINARIES))) -$(addsuffix .exe,$(addprefix tools/win64/,$(BINARIES))): - env GOOS=windows GOARCH=amd64 go build -C extractor -mod=vendor -o ../$@ ./cli/$(basename $(@F)) - -.PHONY: extractor-common extractor extractor-full -extractor-common: codeql-extractor.yml LICENSE ql/lib/go.dbscheme \ - tools/tokenizer.jar $(CODEQL_TOOLS) - rm -rf $(EXTRACTOR_PACK_OUT) - mkdir -p $(EXTRACTOR_PACK_OUT) - cp codeql-extractor.yml LICENSE ql/lib/go.dbscheme ql/lib/go.dbscheme.stats $(EXTRACTOR_PACK_OUT) - mkdir $(EXTRACTOR_PACK_OUT)/tools - cp -r tools/tokenizer.jar $(CODEQL_TOOLS) $(EXTRACTOR_PACK_OUT)/tools - cp -r downgrades $(EXTRACTOR_PACK_OUT) - -extractor: extractor-common tools-codeql - cp -r tools/$(CODEQL_PLATFORM) $(EXTRACTOR_PACK_OUT)/tools - -extractor-full: extractor-common tools-codeql-full - cp -r $(addprefix tools/,linux64 osx64 win64) $(EXTRACTOR_PACK_OUT)/tools - -tools/tokenizer.jar: tools/net/sourceforge/pmd/cpd/GoLanguage.class - jar cf $@ -C tools net - jar uf $@ -C tools opencsv - -tools/net/sourceforge/pmd/cpd/GoLanguage.class: extractor/net/sourceforge/pmd/cpd/GoLanguage.java - javac -cp extractor -d tools $< - rm tools/net/sourceforge/pmd/cpd/AbstractLanguage.class - rm tools/net/sourceforge/pmd/cpd/SourceCode.class - rm tools/net/sourceforge/pmd/cpd/TokenEntry.class - rm tools/net/sourceforge/pmd/cpd/Tokenizer.class - -ql/lib/go.dbscheme: tools/$(CODEQL_PLATFORM)/go-gen-dbscheme$(EXE) - $< $@ +gen: + bazel run :gen build/stats/src.stamp: mkdir -p $(@D)/src @@ -123,7 +62,7 @@ test: all build/testdb/check-upgrade-path codeql test run -j0 ql/test --search-path build/codeql-extractor-go --consistency-queries ql/test/consistency --compilation-cache=$(cache) # use GOOS=linux because GOOS=darwin GOARCH=386 is no longer supported env GOOS=linux GOARCH=386 codeql$(EXE) test run -j0 ql/test/query-tests/Security/CWE-681 --search-path build/codeql-extractor-go --consistency-queries ql/test/consistency --compilation-cache=$(cache) - cd extractor; go test -mod=vendor ./... + cd extractor; bazel test ... bash extractor-smoke-test/test.sh || (echo "Extractor smoke test FAILED"; exit 1) .PHONY: build/testdb/check-upgrade-path From d98ccdfa066544c75c3278c7499e12986cfd11e5 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 26 Apr 2024 16:02:22 +0200 Subject: [PATCH 09/41] Go: update workflow --- .github/workflows/go-tests.yml | 58 +++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 6d9cac5dae9..75c34f4f5de 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -28,13 +28,6 @@ jobs: name: Test Linux (Ubuntu) runs-on: ubuntu-latest-xl steps: - - name: Set up Go ${{ env.GO_VERSION }} - uses: actions/setup-go@v5 - with: - go-version: ${{ env.GO_VERSION }} - cache: false - id: go - - name: Check out code uses: actions/checkout@v4 @@ -47,24 +40,7 @@ jobs: - name: Build run: | - cd go - make - - - name: Check that all Go code is autoformatted - run: | - cd go - make check-formatting - - - name: Compile qhelp files to markdown - run: | - cd go - env QHELP_OUT_DIR=qhelp-out make qhelp-to-markdown - - - name: Upload qhelp markdown - uses: actions/upload-artifact@v3 - with: - name: qhelp-markdown - path: go/qhelp-out/**/*.md + bazel run //go:create-extractor-pack - name: Cache compilation cache id: query-cache @@ -76,3 +52,35 @@ jobs: run: | cd go make test cache="${{ steps.query-cache.outputs.cache-dir }}" + + check-code: + name: Check code + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: Check that all Go code is autoformatted + run: | + cd go + make check-formatting + + - name: Check checked-in generated code + run: | + bazel run //go:gen + git add . + git diff --exit-code HEAD || ( + echo "please run bazel run //go:gen" + exit 1 + ) + + - name: Compile qhelp files to markdown + run: | + cd go + env QHELP_OUT_DIR=qhelp-out make qhelp-to-markdown + + - name: Upload qhelp markdown + uses: actions/upload-artifact@v3 + with: + name: qhelp-markdown + path: go/qhelp-out/**/*.md From 0f387eeac298cb8c30c11a13ba9f9dcd027220e9 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 26 Apr 2024 16:31:04 +0200 Subject: [PATCH 10/41] Go: add vendor update to `//go:gen` --- go/BUILD.bazel | 9 ++++++--- go/extractor/vendor/modules.txt | 1 + go/gen.py | 7 +++++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/go/BUILD.bazel b/go/BUILD.bazel index e769afeea1d..50e2bc0d447 100644 --- a/go/BUILD.bazel +++ b/go/BUILD.bazel @@ -1,15 +1,17 @@ load("@bazel_skylib//rules:native_binary.bzl", "native_binary") load("@gazelle//:def.bzl", "gazelle") load("@rules_pkg//pkg:install.bzl", "pkg_install") -load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup", "pkg_files") +load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files") load("//:defs.bzl", "codeql_platform") gazelle( - name = "_gazelle", + name = "gazelle", + extra_args = ["go/extractor"], ) _gen_binaries = [ - ":_gazelle", + "@rules_go//go", + ":gazelle", "//go/extractor/cli/go-gen-dbscheme", ] @@ -62,6 +64,7 @@ pkg_files( "//go/extractor/cli/go-gen-dbscheme", "//go/extractor/cli/go-tokenizer", ], + attributes = pkg_attributes(mode = "0755"), prefix = "tools/" + codeql_platform, visibility = ["//visibility:public"], ) diff --git a/go/extractor/vendor/modules.txt b/go/extractor/vendor/modules.txt index 5687615f62c..37c4a68d828 100644 --- a/go/extractor/vendor/modules.txt +++ b/go/extractor/vendor/modules.txt @@ -1,3 +1,4 @@ +## workspace # golang.org/x/mod v0.15.0 ## explicit; go 1.18 golang.org/x/mod/internal/lazyregexp diff --git a/go/gen.py b/go/gen.py index 7b7e9d73237..4b4cac0e46f 100644 --- a/go/gen.py +++ b/go/gen.py @@ -7,14 +7,17 @@ this = pathlib.Path(__file__).resolve() go_extractor_dir = this.parent / "extractor" go_dbscheme = this.parent / "ql" / "lib" / "go.dbscheme" r = runfiles.Create() -gazelle, go_gen_dbscheme = map(r.Rlocation, sys.argv[1:]) +go, gazelle, go_gen_dbscheme = map(r.Rlocation, sys.argv[1:]) + +print("updating vendor") +subprocess.check_call([go, "-C", go_extractor_dir, "work", "vendor"]) print("clearing generated BUILD files") for build_file in go_extractor_dir.glob("*/**/BUILD.bazel"): build_file.unlink() print("running gazelle") -subprocess.check_call([gazelle, "go/extractor"]) +subprocess.check_call([gazelle]) print("adding header to generated BUILD files") for build_file in go_extractor_dir.glob("*/**/BUILD.bazel"): From 86d6b8ef21c8a1d82fb23c4530aa31b07de0a305 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 26 Apr 2024 16:39:51 +0200 Subject: [PATCH 11/41] Go: put back go setup --- .github/workflows/go-tests.yml | 47 +++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 75c34f4f5de..5c9204f8405 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -16,9 +16,6 @@ on: - .github/actions/** - codeql-workspace.yml -env: - GO_VERSION: '~1.22.0' - permissions: contents: read @@ -31,6 +28,21 @@ jobs: - name: Check out code uses: actions/checkout@v4 + - name: Get go version + shell: bash + run: | + ( + echo -n "GO_VERSION=" + bazel run @rules_go//go -- version | sed 's/go version go\(\S*\) .*/\1/' + ) | tee -a "$GITHUB_ENV" + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: false + id: go + - name: Set up CodeQL CLI uses: ./.github/actions/fetch-codeql @@ -42,24 +54,6 @@ jobs: run: | bazel run //go:create-extractor-pack - - name: Cache compilation cache - id: query-cache - uses: ./.github/actions/cache-query-compilation - with: - key: go-qltest - - - name: Test - run: | - cd go - make test cache="${{ steps.query-cache.outputs.cache-dir }}" - - check-code: - name: Check code - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v4 - - name: Check that all Go code is autoformatted run: | cd go @@ -84,3 +78,14 @@ jobs: with: name: qhelp-markdown path: go/qhelp-out/**/*.md + + - name: Cache compilation cache + id: query-cache + uses: ./.github/actions/cache-query-compilation + with: + key: go-qltest + + - name: Test + run: | + cd go + make test cache="${{ steps.query-cache.outputs.cache-dir }}" From d66494dcb0b2b23010794a052a2315f19de10f30 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 26 Apr 2024 17:03:56 +0200 Subject: [PATCH 12/41] Go: update `go-tests-other-os.yml` --- .github/workflows/go-tests-other-os.yml | 70 ++++++++----------------- 1 file changed, 21 insertions(+), 49 deletions(-) diff --git a/.github/workflows/go-tests-other-os.yml b/.github/workflows/go-tests-other-os.yml index ded53f868b7..ba3dd0335ad 100644 --- a/.github/workflows/go-tests-other-os.yml +++ b/.github/workflows/go-tests-other-os.yml @@ -7,27 +7,38 @@ on: - .github/workflows/go-tests-other-os.yml - .github/actions/** - codeql-workspace.yml -env: - GO_VERSION: '~1.22.0' permissions: contents: read jobs: - test-mac: - name: Test MacOS - runs-on: macos-latest + test: + name: Test + strategy: + fail-fast: false + matrix: + os: [macos-latest, windows-latest-xl] + if: matrix.os == 'macos-latest' || github.repository_owner == 'github' + runs-on: ${{ matrix.os }} steps: - - name: Set up Go ${{ env.GO_VERSION }} + - name: Check out code + uses: actions/checkout@v4 + + - name: Get go version + shell: bash + run: | + ( + echo -n "GO_VERSION=" + bazel run @rules_go//go -- version | sed 's/go version go\(\S*\) .*/\1/' + ) | tee -a "$GITHUB_ENV" + + - name: Set up Go uses: actions/setup-go@v5 with: go-version: ${{ env.GO_VERSION }} cache: false id: go - - name: Check out code - uses: actions/checkout@v4 - - name: Set up CodeQL CLI uses: ./.github/actions/fetch-codeql @@ -37,8 +48,7 @@ jobs: - name: Build run: | - cd go - make + bazel run //go:create-extractor-pack - name: Cache compilation cache id: query-cache @@ -49,41 +59,3 @@ jobs: run: | cd go make test cache="${{ steps.query-cache.outputs.cache-dir }}" - - test-win: - if: github.repository_owner == 'github' - name: Test Windows - runs-on: windows-latest-xl - steps: - - name: Set up Go ${{ env.GO_VERSION }} - uses: actions/setup-go@v5 - with: - go-version: ${{ env.GO_VERSION }} - cache: false - id: go - - - name: Check out code - uses: actions/checkout@v4 - - - name: Set up CodeQL CLI - uses: ./.github/actions/fetch-codeql - - - name: Enable problem matchers in repository - shell: bash - run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;' - - - name: Build - run: | - cd go - make - - - name: Cache compilation cache - id: query-cache - uses: ./.github/actions/cache-query-compilation - with: - key: go-qltest - - - name: Test - run: | - cd go - make test cache="${{ steps.query-cache.outputs.cache-dir }}" From b0758fd1097191313fc62a95cb61edff994707ee Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 29 Apr 2024 10:29:53 +0200 Subject: [PATCH 13/41] Go: workaround for gazelle on macOS See https://github.com/bazelbuild/bazel-gazelle/issues/1793 for details. --- MODULE.bazel | 6 +++++- go/BUILD.bazel | 13 +++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/MODULE.bazel b/MODULE.bazel index 6daa5a98215..5fd99f76740 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -56,7 +56,11 @@ node.toolchain( use_repo(node, "nodejs", "nodejs_toolchains") go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk") -go_sdk.download(version = "1.22.2") +go_sdk.download(version = "1.22.2") # default + +# following is needed for gazelle on macOS +# see https://github.com/bazelbuild/bazel-gazelle/issues/1793 +go_sdk.download(version = "1.21.9") register_toolchains( "@nodejs_toolchains//:all", diff --git a/go/BUILD.bazel b/go/BUILD.bazel index 50e2bc0d447..41113d45f51 100644 --- a/go/BUILD.bazel +++ b/go/BUILD.bazel @@ -1,12 +1,25 @@ load("@bazel_skylib//rules:native_binary.bzl", "native_binary") load("@gazelle//:def.bzl", "gazelle") +load("@rules_go//go:def.bzl", "go_cross_binary") load("@rules_pkg//pkg:install.bzl", "pkg_install") load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files") load("//:defs.bzl", "codeql_platform") +# following is needed for running gazelle on macOS +# see https://github.com/bazelbuild/bazel-gazelle/issues/1793 +go_cross_binary( + name = "gazelle-1.21.9", + sdk_version = "1.21.9", + target = "@gazelle//cmd/gazelle", +) + gazelle( name = "gazelle", extra_args = ["go/extractor"], + gazelle = select({ + "@platforms//os:macos": ":gazelle-1.21.9", + "//conditions:default": "@gazelle//cmd/gazelle", + }), ) _gen_binaries = [ From 0dfd3367291da2e84b2dbe95f57d3ae054992bf3 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 29 Apr 2024 10:34:22 +0200 Subject: [PATCH 14/41] Go: fix `//go:gen` on windows --- go/gen.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/go/gen.py b/go/gen.py index 4b4cac0e46f..b9394cd15b2 100644 --- a/go/gen.py +++ b/go/gen.py @@ -1,11 +1,17 @@ import sys import pathlib import subprocess +import os from python.runfiles import runfiles -this = pathlib.Path(__file__).resolve() -go_extractor_dir = this.parent / "extractor" -go_dbscheme = this.parent / "ql" / "lib" / "go.dbscheme" +try: + workspace_dir = pathlib.Path(os.environ['BUILD_WORKSPACE_DIRECTORY']) +except KeyError: + print("this should be run with bazel run", file=sys.stderr) + sys.exit(1) + +go_extractor_dir = workspace_dir / "go" / "extractor" +go_dbscheme = workspace_dir / "go" / "ql" / "lib" / "go.dbscheme" r = runfiles.Create() go, gazelle, go_gen_dbscheme = map(r.Rlocation, sys.argv[1:]) From 6ec223c5150efbacdb14157f7e5f0815bac7d62a Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 29 Apr 2024 11:00:21 +0200 Subject: [PATCH 15/41] Go: small cleanup in `Makefile` --- go/Makefile | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/go/Makefile b/go/Makefile index d9ba2c7e2d4..3ebeb69fcaf 100644 --- a/go/Makefile +++ b/go/Makefile @@ -1,19 +1,5 @@ all: gen extractor -ifeq ($(OS),Windows_NT) -EXE = .exe -CODEQL_PLATFORM = win64 -else -EXE = -UNAME_S := $(shell uname -s) -ifeq ($(UNAME_S),Linux) -CODEQL_PLATFORM = linux64 -endif -ifeq ($(UNAME_S),Darwin) -CODEQL_PLATFORM = osx64 -endif -endif - EXTRACTOR_PACK_OUT = build/codeql-extractor-go .PHONY: extractor gen clean autoformat check-formatting From 2f6dd2ab819c59425a4c636cdb65930d6c6405fb Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 29 Apr 2024 11:11:23 +0200 Subject: [PATCH 16/41] Go: refactor workflows with shared action --- .github/workflows/go-tests-other-os.yml | 57 +++++------------- .github/workflows/go-tests.yml | 64 +------------------- go/actions/test/action.yml | 80 +++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 104 deletions(-) create mode 100644 go/actions/test/action.yml diff --git a/.github/workflows/go-tests-other-os.yml b/.github/workflows/go-tests-other-os.yml index ba3dd0335ad..9915b0869db 100644 --- a/.github/workflows/go-tests-other-os.yml +++ b/.github/workflows/go-tests-other-os.yml @@ -12,50 +12,21 @@ permissions: contents: read jobs: - test: - name: Test - strategy: - fail-fast: false - matrix: - os: [macos-latest, windows-latest-xl] - if: matrix.os == 'macos-latest' || github.repository_owner == 'github' - runs-on: ${{ matrix.os }} + test-mac: + name: Test MacOS + runs-on: macos-latest steps: - name: Check out code uses: actions/checkout@v4 + - name: Run tests + uses: ./go/actions/test - - name: Get go version - shell: bash - run: | - ( - echo -n "GO_VERSION=" - bazel run @rules_go//go -- version | sed 's/go version go\(\S*\) .*/\1/' - ) | tee -a "$GITHUB_ENV" - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: ${{ env.GO_VERSION }} - cache: false - id: go - - - name: Set up CodeQL CLI - uses: ./.github/actions/fetch-codeql - - - name: Enable problem matchers in repository - shell: bash - run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;' - - - name: Build - run: | - bazel run //go:create-extractor-pack - - - name: Cache compilation cache - id: query-cache - uses: ./.github/actions/cache-query-compilation - with: - key: go-qltest - - name: Test - run: | - cd go - make test cache="${{ steps.query-cache.outputs.cache-dir }}" + test-win: + if: github.repository_owner == 'github' + name: Test Windows + runs-on: windows-latest-xl + steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Run tests + uses: ./go/actions/test diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 5c9204f8405..63e2b7c4974 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -27,65 +27,7 @@ jobs: steps: - name: Check out code uses: actions/checkout@v4 - - - name: Get go version - shell: bash - run: | - ( - echo -n "GO_VERSION=" - bazel run @rules_go//go -- version | sed 's/go version go\(\S*\) .*/\1/' - ) | tee -a "$GITHUB_ENV" - - - name: Set up Go - uses: actions/setup-go@v5 + - name: Run tests + uses: ./go/actions/test with: - go-version: ${{ env.GO_VERSION }} - cache: false - id: go - - - name: Set up CodeQL CLI - uses: ./.github/actions/fetch-codeql - - - name: Enable problem matchers in repository - shell: bash - run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;' - - - name: Build - run: | - bazel run //go:create-extractor-pack - - - name: Check that all Go code is autoformatted - run: | - cd go - make check-formatting - - - name: Check checked-in generated code - run: | - bazel run //go:gen - git add . - git diff --exit-code HEAD || ( - echo "please run bazel run //go:gen" - exit 1 - ) - - - name: Compile qhelp files to markdown - run: | - cd go - env QHELP_OUT_DIR=qhelp-out make qhelp-to-markdown - - - name: Upload qhelp markdown - uses: actions/upload-artifact@v3 - with: - name: qhelp-markdown - path: go/qhelp-out/**/*.md - - - name: Cache compilation cache - id: query-cache - uses: ./.github/actions/cache-query-compilation - with: - key: go-qltest - - - name: Test - run: | - cd go - make test cache="${{ steps.query-cache.outputs.cache-dir }}" + run-code-checks: true diff --git a/go/actions/test/action.yml b/go/actions/test/action.yml new file mode 100644 index 00000000000..f9bdee5fe0c --- /dev/null +++ b/go/actions/test/action.yml @@ -0,0 +1,80 @@ +name: Test go extractor +description: Run build, QL tests and optionally basic code sanity checks (formatting and generation) +inputs: + run-code-checks: + description: Whether to run formatting, code and qhelp generation checks + required: false + default: false +runs: + using: composite + steps: + - name: Get go version + shell: bash + run: | + ( + echo -n "GO_VERSION=" + bazel run @rules_go//go -- version | sed 's/go version go\(\S*\) .*/\1/' + ) | tee -a "$GITHUB_ENV" + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: false + id: go + + - name: Set up CodeQL CLI + uses: ./.github/actions/fetch-codeql + + - name: Enable problem matchers in repository + shell: bash + run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;' + + - name: Build + shell: bash + run: | + bazel run //go:create-extractor-pack + + - name: Check that all Go code is autoformatted + if: inputs.run-code-checks == 'true' + shell: bash + run: | + cd go + make check-formatting + + - name: Check checked-in generated code + if: inputs.run-code-checks == 'true' + shell: bash + run: | + bazel run //go:gen + git add . + git diff --exit-code HEAD || ( + echo "please run bazel run //go:gen" + exit 1 + ) + + - name: Compile qhelp files to markdown + if: inputs.run-code-checks == 'true' + shell: bash + run: | + cd go + env QHELP_OUT_DIR=qhelp-out make qhelp-to-markdown + + - name: Upload qhelp markdown + if: inputs.run-code-checks == 'true' + uses: actions/upload-artifact@v3 + with: + name: qhelp-markdown + path: go/qhelp-out/**/*.md + + - name: Cache compilation cache + id: query-cache + uses: ./.github/actions/cache-query-compilation + with: + key: go-qltest + + - name: Test + shell: bash + run: | + cd go + make test cache="${{ steps.query-cache.outputs.cache-dir }}" From f0f6c229f6d10c6f3f770777a5bc23b1bd808439 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 29 Apr 2024 11:16:30 +0200 Subject: [PATCH 17/41] Go: fix regex in action for macOS --- go/actions/test/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/actions/test/action.yml b/go/actions/test/action.yml index f9bdee5fe0c..ade22410d95 100644 --- a/go/actions/test/action.yml +++ b/go/actions/test/action.yml @@ -13,7 +13,7 @@ runs: run: | ( echo -n "GO_VERSION=" - bazel run @rules_go//go -- version | sed 's/go version go\(\S*\) .*/\1/' + bazel run @rules_go//go -- version | sed 's/go version go\(.*\) .*/\1/' ) | tee -a "$GITHUB_ENV" - name: Set up Go From 1f78882cdc80011a86dfeb866b08dd46f3004121 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 29 Apr 2024 11:27:16 +0200 Subject: [PATCH 18/41] Go: make windows checks happy --- go/actions/test/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/actions/test/action.yml b/go/actions/test/action.yml index ade22410d95..a351c4382ac 100644 --- a/go/actions/test/action.yml +++ b/go/actions/test/action.yml @@ -33,7 +33,7 @@ runs: - name: Build shell: bash run: | - bazel run //go:create-extractor-pack + bazel run go:create-extractor-pack - name: Check that all Go code is autoformatted if: inputs.run-code-checks == 'true' @@ -46,7 +46,7 @@ runs: if: inputs.run-code-checks == 'true' shell: bash run: | - bazel run //go:gen + bazel run go:gen git add . git diff --exit-code HEAD || ( echo "please run bazel run //go:gen" From 15bb846a5fb9a00748219475e4a2d436181e5e1a Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 29 Apr 2024 12:17:47 +0200 Subject: [PATCH 19/41] Go: add workaround for extractor pack windows installer --- go/BUILD.bazel | 23 +++++++++++++++++++++-- go/create_extractor_pack.py | 16 +++++++++++++--- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/go/BUILD.bazel b/go/BUILD.bazel index 41113d45f51..4251ebd228f 100644 --- a/go/BUILD.bazel +++ b/go/BUILD.bazel @@ -3,6 +3,7 @@ load("@gazelle//:def.bzl", "gazelle") load("@rules_go//go:def.bzl", "go_cross_binary") load("@rules_pkg//pkg:install.bzl", "pkg_install") load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files") +load("@rules_pkg//pkg:zip.bzl", "pkg_zip") load("//:defs.bzl", "codeql_platform") # following is needed for running gazelle on macOS @@ -92,15 +93,33 @@ pkg_filegroup( ) pkg_install( - name = "_create_extractor_pack", + name = "_extractor-pack-installer", srcs = [":extractor-pack"], ) +# rules_pkg installer is currently broken on Windows +# see https://github.com/bazelbuild/rules_pkg/issues/387 +# for now, work around it using an archive +pkg_zip( + name = "_extractor-pack-zip", + srcs = [":extractor-pack"], +) + +alias( + name = "_create-extractor-pack-arg", + actual = select({ + "@platforms//os:windows": ":_extractor-pack-zip", + "//conditions:default": ":_extractor-pack-installer", + }), +) + py_binary( name = "create-extractor-pack", srcs = ["create_extractor_pack.py"], + args = ["$(rlocationpath :_create-extractor-pack-arg)"], + data = [":_create-extractor-pack-arg"], main = "create_extractor_pack.py", - deps = [":_create_extractor_pack"], + deps = ["@rules_python//python/runfiles"], ) native_binary( diff --git a/go/create_extractor_pack.py b/go/create_extractor_pack.py index 08665a2d8dc..a1154a777d8 100644 --- a/go/create_extractor_pack.py +++ b/go/create_extractor_pack.py @@ -2,7 +2,9 @@ import os import pathlib import shutil import sys -from go._create_extractor_pack_install_script import main +import subprocess +import zipfile +from python.runfiles import runfiles try: workspace_dir = pathlib.Path(os.environ['BUILD_WORKSPACE_DIRECTORY']) @@ -11,6 +13,14 @@ except KeyError: sys.exit(1) dest_dir = workspace_dir / 'go' / 'build' / 'codeql-extractor-go' +installer_or_zip = pathlib.Path(runfiles.Create().Rlocation(sys.argv[1])) + shutil.rmtree(dest_dir, ignore_errors=True) -os.environ['DESTDIR'] = str(dest_dir) -main(sys.argv) + +if installer_or_zip.suffix == '.zip': + dest_dir.mkdir() + with zipfile.ZipFile(installer_or_zip) as pack: + pack.extractall(dest_dir) +else: + os.environ['DESTDIR'] = str(dest_dir) + subprocess.check_call([installer_or_zip]) From e7886d0e572a3b13e073dce8a121ef44dd9e6e26 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 29 Apr 2024 13:31:48 +0200 Subject: [PATCH 20/41] Bazel: add empty registry override --- .bazelrc | 3 + .bazelrc.internal | 4 + misc/bazel/registry/AUTHORS | 7 + misc/bazel/registry/LICENSE | 202 ++++++++++++++++++++++++ misc/bazel/registry/NOTICE | 3 + misc/bazel/registry/README.md | 3 + misc/bazel/registry/bazel_registry.json | 3 + misc/bazel/registry/fix.py | 55 +++++++ 8 files changed, 280 insertions(+) create mode 100644 .bazelrc.internal create mode 100644 misc/bazel/registry/AUTHORS create mode 100644 misc/bazel/registry/LICENSE create mode 100644 misc/bazel/registry/NOTICE create mode 100644 misc/bazel/registry/README.md create mode 100644 misc/bazel/registry/bazel_registry.json create mode 100755 misc/bazel/registry/fix.py diff --git a/.bazelrc b/.bazelrc index 12232b4bbd6..0a49f682da3 100644 --- a/.bazelrc +++ b/.bazelrc @@ -14,4 +14,7 @@ build:linux --cxxopt=-std=c++20 build:macos --cxxopt=-std=c++20 --cpu=darwin_x86_64 build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor +common --registry=file:///%workspace%/misc/bazel/registry +common --registry=https://bcr.bazel.build + try-import %workspace%/local.bazelrc diff --git a/.bazelrc.internal b/.bazelrc.internal new file mode 100644 index 00000000000..cdffa9ccdea --- /dev/null +++ b/.bazelrc.internal @@ -0,0 +1,4 @@ +# this file should contain bazel settings required to build things from `semmle-code` + +common --registry=file:///%workspace%/ql/misc/bazel/registry +common --registry=https://bcr.bazel.build diff --git a/misc/bazel/registry/AUTHORS b/misc/bazel/registry/AUTHORS new file mode 100644 index 00000000000..42818b292e7 --- /dev/null +++ b/misc/bazel/registry/AUTHORS @@ -0,0 +1,7 @@ +# This is the list of Bazel's significant contributors. +# +# This does not necessarily list everyone who has contributed code, +# especially since many employees of one corporation may be contributing. +# To see the full list of contributors, see the revision history in +# source control. +Google LLC diff --git a/misc/bazel/registry/LICENSE b/misc/bazel/registry/LICENSE new file mode 100644 index 00000000000..d6456956733 --- /dev/null +++ b/misc/bazel/registry/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/misc/bazel/registry/NOTICE b/misc/bazel/registry/NOTICE new file mode 100644 index 00000000000..95329ae18f6 --- /dev/null +++ b/misc/bazel/registry/NOTICE @@ -0,0 +1,3 @@ +The files in this directory where originally taken from http://github.com/bazelbuild/bazel-central-registry and are +a derivative work thereof, distributed under the Apache 2.0 license, with the following exceptions: +* the `fix.py` file was added under the MIT license as the rest of the `codeql` repository. diff --git a/misc/bazel/registry/README.md b/misc/bazel/registry/README.md new file mode 100644 index 00000000000..5d1723d0eac --- /dev/null +++ b/misc/bazel/registry/README.md @@ -0,0 +1,3 @@ +Versions to be patched can be taken from https://github.com/bazelbuild/bazel-central-repository. After adding patches +inside `//patches`, and eventually renaming ``, run [`fix.py`](./fix.py) to align all metadata +to the renamed version and added patches. diff --git a/misc/bazel/registry/bazel_registry.json b/misc/bazel/registry/bazel_registry.json new file mode 100644 index 00000000000..ea3f94f7a1e --- /dev/null +++ b/misc/bazel/registry/bazel_registry.json @@ -0,0 +1,3 @@ +{ + "mirrors": [] +} diff --git a/misc/bazel/registry/fix.py b/misc/bazel/registry/fix.py new file mode 100755 index 00000000000..a2b947e19e2 --- /dev/null +++ b/misc/bazel/registry/fix.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2024 GitHub, Inc. + +""" +Fix metadata in overridden registry, updating `metadata.json` to list correct versions and `source.json` +to list correct patches with sha256 hashes. +""" + +import pathlib +import json +import base64 +import hashlib +import re + +this_dir = pathlib.Path(__file__).resolve().parent + + +def sha256(file): + with open(file, 'rb') as input: + hash = hashlib.sha256(input.read()).digest() + hash = base64.b64encode(hash).decode() + return f"sha256-{hash}" + + +def patch_file(file, f): + try: + data = file.read_text() + except FileNotFoundError: + data = None + file.write_text(f(data)) + + +def patch_json(file, **kwargs): + def update(data): + data = json.loads(data) if data else {} + data.update(kwargs) + return json.dumps(data, indent=4) + "\n" + + patch_file(file, update) + + +for entry in this_dir.joinpath("modules").iterdir(): + if not entry.is_dir(): + continue + versions = [e for e in entry.iterdir() if e.is_dir()] + + patch_json(entry / "metadata.json", versions=[v.name for v in versions]) + + for version in versions: + patch_json(version / "source.json", patches={ + p.name: sha256(p) for p in version.joinpath("patches").iterdir() + }) + patch_file(version / "MODULE.bazel", + lambda s: re.sub(r'''version\s*=\s*['"].*['"]''', f'version = "{version.name}"', s, 1)) From cb85a756a053b4f7526e3ef7ab88e4c044baa39d Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 29 Apr 2024 13:42:05 +0200 Subject: [PATCH 21/41] Bazel: patch `apple_support` to avoid registering its toolchain This is done in order to avoid requiring a full Xcode installation, but still being able to use other `apple_support` facilities, like `universal_binary`. --- MODULE.bazel | 2 +- .../1.15.1-codeql.1/MODULE.bazel | 17 ++++++++++++++++ .../1.15.1-codeql.1/patches/module.patch | 20 +++++++++++++++++++ .../apple_support/1.15.1-codeql.1/source.json | 9 +++++++++ .../modules/apple_support/metadata.json | 5 +++++ 5 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/MODULE.bazel create mode 100644 misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/patches/module.patch create mode 100644 misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/source.json create mode 100644 misc/bazel/registry/modules/apple_support/metadata.json diff --git a/MODULE.bazel b/MODULE.bazel index 5fd99f76740..875e61da383 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -13,7 +13,7 @@ local_path_override( # see https://registry.bazel.build/ for a list of available packages -bazel_dep(name = "apple_support", version = "1.15.1") +bazel_dep(name = "apple_support", version = "1.15.1-codeql.1") bazel_dep(name = "platforms", version = "0.0.9") bazel_dep(name = "rules_go", version = "0.47.0") bazel_dep(name = "rules_pkg", version = "0.10.1") diff --git a/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/MODULE.bazel b/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/MODULE.bazel new file mode 100644 index 00000000000..a58d520fee2 --- /dev/null +++ b/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/MODULE.bazel @@ -0,0 +1,17 @@ +module( + name = "apple_support", + version = "1.15.1-codeql.1", + bazel_compatibility = [">=6.0.0"], + compatibility_level = 1, + repo_name = "build_bazel_apple_support", +) + +bazel_dep(name = "bazel_skylib", version = "1.3.0") +bazel_dep(name = "platforms", version = "0.0.9") + +bazel_dep( + name = "stardoc", + version = "0.6.2", + dev_dependency = True, + repo_name = "io_bazel_stardoc", +) diff --git a/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/patches/module.patch b/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/patches/module.patch new file mode 100644 index 00000000000..59fc49ec7b3 --- /dev/null +++ b/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/patches/module.patch @@ -0,0 +1,20 @@ +diff --git a/MODULE.bazel b/MODULE.bazel +index 6b06c3b..99bc7c6 100644 +--- a/MODULE.bazel ++++ b/MODULE.bazel +@@ -1,6 +1,6 @@ + module( + name = "apple_support", +- version = "0", ++ version = "1.15.1-codeql.1", + bazel_compatibility = [">=6.0.0"], + compatibility_level = 1, + repo_name = "build_bazel_apple_support", +@@ -16,7 +16,3 @@ bazel_dep( + repo_name = "io_bazel_stardoc", + ) + +-apple_cc_configure = use_extension("//crosstool:setup.bzl", "apple_cc_configure_extension") +-use_repo(apple_cc_configure, "local_config_apple_cc", "local_config_apple_cc_toolchains") +- +-register_toolchains("@local_config_apple_cc_toolchains//:all") diff --git a/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/source.json b/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/source.json new file mode 100644 index 00000000000..57a1f2137fe --- /dev/null +++ b/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/source.json @@ -0,0 +1,9 @@ +{ + "integrity": "sha256-xLsrc2fEhDgjAK7nW+WYuS+EeJb7MbvSLzojRq32aoA=", + "strip_prefix": "", + "url": "https://github.com/bazelbuild/apple_support/releases/download/1.15.1/apple_support.1.15.1.tar.gz", + "patches": { + "module.patch": "sha256-K06B2W9t6nKcU8S5u6cWeNIdw/vGWWKAoJdGiI8CSS0=" + }, + "patch_strip": 1 +} diff --git a/misc/bazel/registry/modules/apple_support/metadata.json b/misc/bazel/registry/modules/apple_support/metadata.json new file mode 100644 index 00000000000..8a0e54c9abd --- /dev/null +++ b/misc/bazel/registry/modules/apple_support/metadata.json @@ -0,0 +1,5 @@ +{ + "versions": [ + "1.15.1-codeql.1" + ] +} From 608791fd7f99166b6f6daba5b27f9abb7c37bac9 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 2 May 2024 09:13:40 +0200 Subject: [PATCH 22/41] Bazel/Go: use native cross compilation for fat binaries --- go/rules.bzl | 40 +++++++++++++++++++++++++++++---- misc/bazel/universal_binary.bzl | 24 -------------------- 2 files changed, 36 insertions(+), 28 deletions(-) delete mode 100644 misc/bazel/universal_binary.bzl diff --git a/go/rules.bzl b/go/rules.bzl index 4ef798001d2..e26dd57bf44 100644 --- a/go/rules.bzl +++ b/go/rules.bzl @@ -1,5 +1,37 @@ -load("@rules_go//go:def.bzl", "go_binary") -load("//misc/bazel:universal_binary.bzl", "wrap_as_universal_binary") +load("@rules_go//go:def.bzl", "go_binary", "go_cross_binary") -def codeql_go_binary(**kwargs): - wrap_as_universal_binary(go_binary, **kwargs) +def codeql_go_binary(*, name, visibility = None, **kwargs): + def internal(prefix = "internal"): + return "%s/%s" % (prefix, name) + + go_binary( + name = internal(), + visibility = ["//visibility:private"], + **kwargs + ) + macos_targets = ("darwin_arm64", "darwin_amd64") + for target in macos_targets: + go_cross_binary( + name = internal(target), + platform = "@rules_go//go/toolchain:%s" % target, + target = internal(), + target_compatible_with = ["@platforms//os:macos"], + visibility = ["//visibility:private"], + ) + native.genrule( + name = internal("universal"), + outs = [internal("universal_")], + srcs = [internal(t) for t in macos_targets], + target_compatible_with = ["@platforms//os:macos"], + executable = True, + visibility = ["//visibility:private"], + cmd = "lipo -create $(SRCS) -output $@", + ) + native.alias( + name = name, + actual = select({ + "@platforms//os:macos": internal("universal"), + "//conditions:default": internal(), + }), + visibility = visibility, + ) diff --git a/misc/bazel/universal_binary.bzl b/misc/bazel/universal_binary.bzl deleted file mode 100644 index 85881356d0e..00000000000 --- a/misc/bazel/universal_binary.bzl +++ /dev/null @@ -1,24 +0,0 @@ -load("@apple_support//rules:universal_binary.bzl", _universal_binary = "universal_binary") - -def wrap_as_universal_binary(rule, *, name, visibility = None, **kwargs): - internal_name = "internal/%s" % name - universal_name = "universal/%s" % name - rule( - name = internal_name, - visibility = ["//visibility:private"], - **kwargs - ) - _universal_binary( - name = universal_name, - target_compatible_with = ["@platforms//os:macos"], - binary = internal_name, - visibility = ["//visibility:private"], - ) - native.alias( - name = name, - actual = select({ - "@platforms//os:macos": universal_name, - "//conditions:default": internal_name, - }), - visibility = visibility, - ) From 94212d103ec3cffd91f5efcffa668db36f99b7a7 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 2 May 2024 09:14:15 +0200 Subject: [PATCH 23/41] Bazel/Go: remove `apple_support` This was actually unnecessary, and requried a full Xcode installation that we'd rather avoid. --- MODULE.bazel | 5 ----- go/BUILD.bazel | 13 ------------ .../1.15.1-codeql.1/MODULE.bazel | 17 ---------------- .../1.15.1-codeql.1/patches/module.patch | 20 ------------------- .../apple_support/1.15.1-codeql.1/source.json | 9 --------- .../modules/apple_support/metadata.json | 5 ----- 6 files changed, 69 deletions(-) delete mode 100644 misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/MODULE.bazel delete mode 100644 misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/patches/module.patch delete mode 100644 misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/source.json delete mode 100644 misc/bazel/registry/modules/apple_support/metadata.json diff --git a/MODULE.bazel b/MODULE.bazel index 875e61da383..d069d320f07 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -13,7 +13,6 @@ local_path_override( # see https://registry.bazel.build/ for a list of available packages -bazel_dep(name = "apple_support", version = "1.15.1-codeql.1") bazel_dep(name = "platforms", version = "0.0.9") bazel_dep(name = "rules_go", version = "0.47.0") bazel_dep(name = "rules_pkg", version = "0.10.1") @@ -58,10 +57,6 @@ use_repo(node, "nodejs", "nodejs_toolchains") go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk") go_sdk.download(version = "1.22.2") # default -# following is needed for gazelle on macOS -# see https://github.com/bazelbuild/bazel-gazelle/issues/1793 -go_sdk.download(version = "1.21.9") - register_toolchains( "@nodejs_toolchains//:all", ) diff --git a/go/BUILD.bazel b/go/BUILD.bazel index 4251ebd228f..6c80e7cb258 100644 --- a/go/BUILD.bazel +++ b/go/BUILD.bazel @@ -1,26 +1,13 @@ load("@bazel_skylib//rules:native_binary.bzl", "native_binary") load("@gazelle//:def.bzl", "gazelle") -load("@rules_go//go:def.bzl", "go_cross_binary") load("@rules_pkg//pkg:install.bzl", "pkg_install") load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files") load("@rules_pkg//pkg:zip.bzl", "pkg_zip") load("//:defs.bzl", "codeql_platform") -# following is needed for running gazelle on macOS -# see https://github.com/bazelbuild/bazel-gazelle/issues/1793 -go_cross_binary( - name = "gazelle-1.21.9", - sdk_version = "1.21.9", - target = "@gazelle//cmd/gazelle", -) - gazelle( name = "gazelle", extra_args = ["go/extractor"], - gazelle = select({ - "@platforms//os:macos": ":gazelle-1.21.9", - "//conditions:default": "@gazelle//cmd/gazelle", - }), ) _gen_binaries = [ diff --git a/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/MODULE.bazel b/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/MODULE.bazel deleted file mode 100644 index a58d520fee2..00000000000 --- a/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/MODULE.bazel +++ /dev/null @@ -1,17 +0,0 @@ -module( - name = "apple_support", - version = "1.15.1-codeql.1", - bazel_compatibility = [">=6.0.0"], - compatibility_level = 1, - repo_name = "build_bazel_apple_support", -) - -bazel_dep(name = "bazel_skylib", version = "1.3.0") -bazel_dep(name = "platforms", version = "0.0.9") - -bazel_dep( - name = "stardoc", - version = "0.6.2", - dev_dependency = True, - repo_name = "io_bazel_stardoc", -) diff --git a/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/patches/module.patch b/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/patches/module.patch deleted file mode 100644 index 59fc49ec7b3..00000000000 --- a/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/patches/module.patch +++ /dev/null @@ -1,20 +0,0 @@ -diff --git a/MODULE.bazel b/MODULE.bazel -index 6b06c3b..99bc7c6 100644 ---- a/MODULE.bazel -+++ b/MODULE.bazel -@@ -1,6 +1,6 @@ - module( - name = "apple_support", -- version = "0", -+ version = "1.15.1-codeql.1", - bazel_compatibility = [">=6.0.0"], - compatibility_level = 1, - repo_name = "build_bazel_apple_support", -@@ -16,7 +16,3 @@ bazel_dep( - repo_name = "io_bazel_stardoc", - ) - --apple_cc_configure = use_extension("//crosstool:setup.bzl", "apple_cc_configure_extension") --use_repo(apple_cc_configure, "local_config_apple_cc", "local_config_apple_cc_toolchains") -- --register_toolchains("@local_config_apple_cc_toolchains//:all") diff --git a/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/source.json b/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/source.json deleted file mode 100644 index 57a1f2137fe..00000000000 --- a/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/source.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "integrity": "sha256-xLsrc2fEhDgjAK7nW+WYuS+EeJb7MbvSLzojRq32aoA=", - "strip_prefix": "", - "url": "https://github.com/bazelbuild/apple_support/releases/download/1.15.1/apple_support.1.15.1.tar.gz", - "patches": { - "module.patch": "sha256-K06B2W9t6nKcU8S5u6cWeNIdw/vGWWKAoJdGiI8CSS0=" - }, - "patch_strip": 1 -} diff --git a/misc/bazel/registry/modules/apple_support/metadata.json b/misc/bazel/registry/modules/apple_support/metadata.json deleted file mode 100644 index 8a0e54c9abd..00000000000 --- a/misc/bazel/registry/modules/apple_support/metadata.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "versions": [ - "1.15.1-codeql.1" - ] -} From 12b9b805e2f40fd72bf30c0c8178c6b04e647891 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 2 May 2024 12:37:47 +0200 Subject: [PATCH 24/41] Go: revert changes to `make` and CI to postpone them in a separate PR --- .github/workflows/go-tests-other-os.yml | 65 ++++++++++++++++-- .github/workflows/go-tests.yml | 51 +++++++++++++- go/Makefile | 89 +++++++++++++++++++++++-- 3 files changed, 191 insertions(+), 14 deletions(-) diff --git a/.github/workflows/go-tests-other-os.yml b/.github/workflows/go-tests-other-os.yml index 9915b0869db..ded53f868b7 100644 --- a/.github/workflows/go-tests-other-os.yml +++ b/.github/workflows/go-tests-other-os.yml @@ -7,6 +7,8 @@ on: - .github/workflows/go-tests-other-os.yml - .github/actions/** - codeql-workspace.yml +env: + GO_VERSION: '~1.22.0' permissions: contents: read @@ -16,17 +18,72 @@ jobs: name: Test MacOS runs-on: macos-latest steps: + - name: Set up Go ${{ env.GO_VERSION }} + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: false + id: go + - name: Check out code uses: actions/checkout@v4 - - name: Run tests - uses: ./go/actions/test + + - name: Set up CodeQL CLI + uses: ./.github/actions/fetch-codeql + + - name: Enable problem matchers in repository + shell: bash + run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;' + + - name: Build + run: | + cd go + make + + - name: Cache compilation cache + id: query-cache + uses: ./.github/actions/cache-query-compilation + with: + key: go-qltest + - name: Test + run: | + cd go + make test cache="${{ steps.query-cache.outputs.cache-dir }}" test-win: if: github.repository_owner == 'github' name: Test Windows runs-on: windows-latest-xl steps: + - name: Set up Go ${{ env.GO_VERSION }} + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: false + id: go + - name: Check out code uses: actions/checkout@v4 - - name: Run tests - uses: ./go/actions/test + + - name: Set up CodeQL CLI + uses: ./.github/actions/fetch-codeql + + - name: Enable problem matchers in repository + shell: bash + run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;' + + - name: Build + run: | + cd go + make + + - name: Cache compilation cache + id: query-cache + uses: ./.github/actions/cache-query-compilation + with: + key: go-qltest + + - name: Test + run: | + cd go + make test cache="${{ steps.query-cache.outputs.cache-dir }}" diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 63e2b7c4974..6d9cac5dae9 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -16,6 +16,9 @@ on: - .github/actions/** - codeql-workspace.yml +env: + GO_VERSION: '~1.22.0' + permissions: contents: read @@ -25,9 +28,51 @@ jobs: name: Test Linux (Ubuntu) runs-on: ubuntu-latest-xl steps: + - name: Set up Go ${{ env.GO_VERSION }} + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: false + id: go + - name: Check out code uses: actions/checkout@v4 - - name: Run tests - uses: ./go/actions/test + + - name: Set up CodeQL CLI + uses: ./.github/actions/fetch-codeql + + - name: Enable problem matchers in repository + shell: bash + run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;' + + - name: Build + run: | + cd go + make + + - name: Check that all Go code is autoformatted + run: | + cd go + make check-formatting + + - name: Compile qhelp files to markdown + run: | + cd go + env QHELP_OUT_DIR=qhelp-out make qhelp-to-markdown + + - name: Upload qhelp markdown + uses: actions/upload-artifact@v3 with: - run-code-checks: true + name: qhelp-markdown + path: go/qhelp-out/**/*.md + + - name: Cache compilation cache + id: query-cache + uses: ./.github/actions/cache-query-compilation + with: + key: go-qltest + + - name: Test + run: | + cd go + make test cache="${{ steps.query-cache.outputs.cache-dir }}" diff --git a/go/Makefile b/go/Makefile index 3ebeb69fcaf..d0289a093a5 100644 --- a/go/Makefile +++ b/go/Makefile @@ -1,10 +1,30 @@ -all: gen extractor +all: extractor ql/lib/go.dbscheme + +ifeq ($(OS),Windows_NT) +EXE = .exe +CODEQL_PLATFORM = win64 +else +EXE = +UNAME_S := $(shell uname -s) +ifeq ($(UNAME_S),Linux) +CODEQL_PLATFORM = linux64 +endif +ifeq ($(UNAME_S),Darwin) +CODEQL_PLATFORM = osx64 +endif +endif + +CODEQL_TOOLS = $(addprefix codeql-tools/,autobuild.cmd autobuild.sh baseline-config-empty.json baseline-config-vendor.json configure-baseline.cmd configure-baseline.sh identify-environment.cmd identify-environment.sh index.cmd index.sh pre-finalize.cmd pre-finalize.sh tracing-config.lua) EXTRACTOR_PACK_OUT = build/codeql-extractor-go -.PHONY: extractor gen clean autoformat check-formatting +BINARIES = go-extractor go-tokenizer go-autobuilder go-build-runner go-bootstrap go-gen-dbscheme + +.PHONY: tools tools-codeql tools-codeql-full clean autoformat \ + tools-linux64 tools-osx64 tools-win64 check-formatting clean: + rm -rf tools/bin tools/linux64 tools/osx64 tools/win64 tools/net tools/opencsv rm -rf $(EXTRACTOR_PACK_OUT) build/stats build/testdb autoformat: @@ -27,11 +47,66 @@ endif qhelp-to-markdown: scripts/qhelp-to-markdown.sh ql/src "$(QHELP_OUT_DIR)" -extractor: - bazel run :create-extractor-pack +tools: tools-codeql tools/tokenizer.jar -gen: - bazel run :gen +.PHONY: $(addsuffix $(EXE),$(addprefix tools/bin/,$(BINARIES))) +$(addsuffix $(EXE),$(addprefix tools/bin/,$(BINARIES))): + go build -C extractor -mod=vendor -o ../$@ ./cli/$(basename $(@F)) + +tools-codeql: tools-$(CODEQL_PLATFORM) + +tools-codeql-full: tools-linux64 tools-osx64 tools-win64 + +tools-linux64: $(addprefix tools/linux64/,$(BINARIES)) + +.PHONY: $(addprefix tools/linux64/,$(BINARIES)) +$(addprefix tools/linux64/,$(BINARIES)): + GOOS=linux GOARCH=amd64 go build -C extractor -mod=vendor -o ../$@ ./cli/$(@F) + +tools-osx64: $(addprefix tools/osx64/,$(BINARIES)) + +.PHONY: $(addprefix tools/osx64/,$(BINARIES)) +$(addprefix tools/osx64/,$(BINARIES)): + GOOS=darwin GOARCH=amd64 go build -C extractor -mod=vendor -o ../$@.amd64 ./cli/$(@F) + GOOS=darwin GOARCH=arm64 go build -C extractor -mod=vendor -o ../$@.arm64 ./cli/$(@F) + lipo -create $@.amd64 $@.arm64 -output $@ + rm $@.amd64 $@.arm64 + +tools-win64: $(addsuffix .exe,$(addprefix tools/win64/,$(BINARIES))) + +.PHONY: $(addsuffix .exe,$(addprefix tools/win64/,$(BINARIES))) +$(addsuffix .exe,$(addprefix tools/win64/,$(BINARIES))): + env GOOS=windows GOARCH=amd64 go build -C extractor -mod=vendor -o ../$@ ./cli/$(basename $(@F)) + +.PHONY: extractor-common extractor extractor-full +extractor-common: codeql-extractor.yml LICENSE ql/lib/go.dbscheme \ + tools/tokenizer.jar $(CODEQL_TOOLS) + rm -rf $(EXTRACTOR_PACK_OUT) + mkdir -p $(EXTRACTOR_PACK_OUT) + cp codeql-extractor.yml LICENSE ql/lib/go.dbscheme ql/lib/go.dbscheme.stats $(EXTRACTOR_PACK_OUT) + mkdir $(EXTRACTOR_PACK_OUT)/tools + cp -r tools/tokenizer.jar $(CODEQL_TOOLS) $(EXTRACTOR_PACK_OUT)/tools + cp -r downgrades $(EXTRACTOR_PACK_OUT) + +extractor: extractor-common tools-codeql + cp -r tools/$(CODEQL_PLATFORM) $(EXTRACTOR_PACK_OUT)/tools + +extractor-full: extractor-common tools-codeql-full + cp -r $(addprefix tools/,linux64 osx64 win64) $(EXTRACTOR_PACK_OUT)/tools + +tools/tokenizer.jar: tools/net/sourceforge/pmd/cpd/GoLanguage.class + jar cf $@ -C tools net + jar uf $@ -C tools opencsv + +tools/net/sourceforge/pmd/cpd/GoLanguage.class: extractor/net/sourceforge/pmd/cpd/GoLanguage.java + javac -cp extractor -d tools $< + rm tools/net/sourceforge/pmd/cpd/AbstractLanguage.class + rm tools/net/sourceforge/pmd/cpd/SourceCode.class + rm tools/net/sourceforge/pmd/cpd/TokenEntry.class + rm tools/net/sourceforge/pmd/cpd/Tokenizer.class + +ql/lib/go.dbscheme: tools/$(CODEQL_PLATFORM)/go-gen-dbscheme$(EXE) + $< $@ build/stats/src.stamp: mkdir -p $(@D)/src @@ -48,7 +123,7 @@ test: all build/testdb/check-upgrade-path codeql test run -j0 ql/test --search-path build/codeql-extractor-go --consistency-queries ql/test/consistency --compilation-cache=$(cache) # use GOOS=linux because GOOS=darwin GOARCH=386 is no longer supported env GOOS=linux GOARCH=386 codeql$(EXE) test run -j0 ql/test/query-tests/Security/CWE-681 --search-path build/codeql-extractor-go --consistency-queries ql/test/consistency --compilation-cache=$(cache) - cd extractor; bazel test ... + cd extractor; go test -mod=vendor ./... bash extractor-smoke-test/test.sh || (echo "Extractor smoke test FAILED"; exit 1) .PHONY: build/testdb/check-upgrade-path From 9055d9567aa9e850e6c7836aa7086c847192d588 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 2 May 2024 15:08:51 +0200 Subject: [PATCH 25/41] Go: remove unused action (will be re-added later) --- go/actions/test/action.yml | 80 -------------------------------------- 1 file changed, 80 deletions(-) delete mode 100644 go/actions/test/action.yml diff --git a/go/actions/test/action.yml b/go/actions/test/action.yml deleted file mode 100644 index a351c4382ac..00000000000 --- a/go/actions/test/action.yml +++ /dev/null @@ -1,80 +0,0 @@ -name: Test go extractor -description: Run build, QL tests and optionally basic code sanity checks (formatting and generation) -inputs: - run-code-checks: - description: Whether to run formatting, code and qhelp generation checks - required: false - default: false -runs: - using: composite - steps: - - name: Get go version - shell: bash - run: | - ( - echo -n "GO_VERSION=" - bazel run @rules_go//go -- version | sed 's/go version go\(.*\) .*/\1/' - ) | tee -a "$GITHUB_ENV" - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: ${{ env.GO_VERSION }} - cache: false - id: go - - - name: Set up CodeQL CLI - uses: ./.github/actions/fetch-codeql - - - name: Enable problem matchers in repository - shell: bash - run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;' - - - name: Build - shell: bash - run: | - bazel run go:create-extractor-pack - - - name: Check that all Go code is autoformatted - if: inputs.run-code-checks == 'true' - shell: bash - run: | - cd go - make check-formatting - - - name: Check checked-in generated code - if: inputs.run-code-checks == 'true' - shell: bash - run: | - bazel run go:gen - git add . - git diff --exit-code HEAD || ( - echo "please run bazel run //go:gen" - exit 1 - ) - - - name: Compile qhelp files to markdown - if: inputs.run-code-checks == 'true' - shell: bash - run: | - cd go - env QHELP_OUT_DIR=qhelp-out make qhelp-to-markdown - - - name: Upload qhelp markdown - if: inputs.run-code-checks == 'true' - uses: actions/upload-artifact@v3 - with: - name: qhelp-markdown - path: go/qhelp-out/**/*.md - - - name: Cache compilation cache - id: query-cache - uses: ./.github/actions/cache-query-compilation - with: - key: go-qltest - - - name: Test - shell: bash - run: | - cd go - make test cache="${{ steps.query-cache.outputs.cache-dir }}" From ca2d94b297820ff93ef072419c19de31d6c6dd57 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 2 May 2024 15:15:45 +0200 Subject: [PATCH 26/41] Fix go pattern in `.pre-commit-config.yaml` --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 055cab4ac05..5a0ccc4938f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: - id: go-gen name: Check checked in generated files in go - files: go/.* + files: ^go/.* language: system entry: bazel run //go:gen pass_filenames: false From 318d954536bfca389d3c00812a84ff81f99ff64c Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 2 May 2024 15:38:47 +0200 Subject: [PATCH 27/41] Go: make `//go:gen` not clear by default, and clean on `--force` --- go/gen.py | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/go/gen.py b/go/gen.py index b9394cd15b2..ebb57b2a685 100644 --- a/go/gen.py +++ b/go/gen.py @@ -2,8 +2,18 @@ import sys import pathlib import subprocess import os +import argparse +import shutil from python.runfiles import runfiles +def options(): + p = argparse.ArgumentParser(description="Update generated checked in files in the Go pack") + p.add_argument("--force", "-f", action="store_true", help="Regenerate all files from scratch rather than updating them") + p.add_argument("generators", nargs=3) + return p.parse_args() + +opts = options() + try: workspace_dir = pathlib.Path(os.environ['BUILD_WORKSPACE_DIRECTORY']) except KeyError: @@ -13,20 +23,34 @@ except KeyError: go_extractor_dir = workspace_dir / "go" / "extractor" go_dbscheme = workspace_dir / "go" / "ql" / "lib" / "go.dbscheme" r = runfiles.Create() -go, gazelle, go_gen_dbscheme = map(r.Rlocation, sys.argv[1:]) +go, gazelle, go_gen_dbscheme = map(r.Rlocation, opts.generators) -print("updating vendor") + +if opts.force: + print("clearing vendor directory") + shutil.rmtree(go_extractor_dir / "vendor") + +existing_build_files = set(go_extractor_dir.glob("*/**/BUILD.bazel")) + +print("updating vendor directory") subprocess.check_call([go, "-C", go_extractor_dir, "work", "vendor"]) -print("clearing generated BUILD files") -for build_file in go_extractor_dir.glob("*/**/BUILD.bazel"): - build_file.unlink() +if opts.force: + print("clearing generated BUILD files") + for build_file in existing_build_files: + build_file.unlink() print("running gazelle") subprocess.check_call([gazelle]) +build_files_to_update = set(go_extractor_dir.glob("*/**/BUILD.bazel")) +if not opts.force: + build_files_to_update -= existing_build_files + # these are always refreshed + build_files_to_update.update(go_extractor_dir.glob("vendor/**/BUILD.bazel")) + print("adding header to generated BUILD files") -for build_file in go_extractor_dir.glob("*/**/BUILD.bazel"): +for build_file in build_files_to_update: contents = build_file.read_text() build_file.write_text(f"# generated running `bazel run //go/gazelle`, do not edit\n\n{contents}") From 4ae82ac2150cd2c90cb708f1e810381367fb0fa7 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 2 May 2024 15:40:13 +0200 Subject: [PATCH 28/41] Go: add explanatory comment to `extractor` `BUILD` file --- go/extractor/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/go/extractor/BUILD.bazel b/go/extractor/BUILD.bazel index cb7bc3ac8ef..32eaa8fda74 100644 --- a/go/extractor/BUILD.bazel +++ b/go/extractor/BUILD.bazel @@ -4,6 +4,7 @@ load("@rules_pkg//pkg:mappings.bzl", "pkg_files") # gazelle:prefix github.com/github/codeql-go/extractor # gazelle:map_kind go_binary codeql_go_binary //go:rules.bzl +# following target is kept up to date by `bazel run //go:gen`, do not edit directly go_library( name = "extractor", srcs = [ From 0bc6934bfc210d7efc2f87966ab13cae710a82a0 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 2 May 2024 15:44:06 +0200 Subject: [PATCH 29/41] Go: rename `pkg_files` to something less confusing --- go/codeql-tools/BUILD.bazel | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/go/codeql-tools/BUILD.bazel b/go/codeql-tools/BUILD.bazel index 4e839b4774b..8c3946b24ad 100644 --- a/go/codeql-tools/BUILD.bazel +++ b/go/codeql-tools/BUILD.bazel @@ -1,13 +1,13 @@ load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files") pkg_files( - name = "executables", + name = "sh-files", srcs = glob(["*.sh"]), attributes = pkg_attributes(mode = "0755"), ) pkg_files( - name = "non-executables", + name = "non-sh-files", srcs = glob( ["*"], exclude = [ @@ -20,8 +20,8 @@ pkg_files( pkg_filegroup( name = "codeql-tools", srcs = [ - ":executables", - ":non-executables", + ":non-sh-files", + ":sh-files", ], prefix = "tools", visibility = ["//go:__pkg__"], From abcd9165b4b9ded2c2b9c297502df24529f95561 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 2 May 2024 16:08:17 +0200 Subject: [PATCH 30/41] Go: write test go runtime version in a specific file --- .github/workflows/go-tests-other-os.yml | 10 ++++------ .github/workflows/go-tests.yml | 7 ++----- go/ql/test/go.mod | 4 ---- go/test-runtime-version/go.work | 1 + 4 files changed, 7 insertions(+), 15 deletions(-) delete mode 100644 go/ql/test/go.mod create mode 100644 go/test-runtime-version/go.work diff --git a/.github/workflows/go-tests-other-os.yml b/.github/workflows/go-tests-other-os.yml index ded53f868b7..d38fd18f272 100644 --- a/.github/workflows/go-tests-other-os.yml +++ b/.github/workflows/go-tests-other-os.yml @@ -7,8 +7,6 @@ on: - .github/workflows/go-tests-other-os.yml - .github/actions/** - codeql-workspace.yml -env: - GO_VERSION: '~1.22.0' permissions: contents: read @@ -18,10 +16,10 @@ jobs: name: Test MacOS runs-on: macos-latest steps: - - name: Set up Go ${{ env.GO_VERSION }} + - name: Set up Go uses: actions/setup-go@v5 with: - go-version: ${{ env.GO_VERSION }} + go-version-file: go/test-runtime-version/go.work cache: false id: go @@ -55,10 +53,10 @@ jobs: name: Test Windows runs-on: windows-latest-xl steps: - - name: Set up Go ${{ env.GO_VERSION }} + - name: Set up Go uses: actions/setup-go@v5 with: - go-version: ${{ env.GO_VERSION }} + go-version-file: go/test-runtime-version/go.work cache: false id: go diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 6d9cac5dae9..60182bb2c81 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -16,9 +16,6 @@ on: - .github/actions/** - codeql-workspace.yml -env: - GO_VERSION: '~1.22.0' - permissions: contents: read @@ -28,10 +25,10 @@ jobs: name: Test Linux (Ubuntu) runs-on: ubuntu-latest-xl steps: - - name: Set up Go ${{ env.GO_VERSION }} + - name: Set up Go uses: actions/setup-go@v5 with: - go-version: ${{ env.GO_VERSION }} + go-version-file: go/test-runtime-version/go.work cache: false id: go diff --git a/go/ql/test/go.mod b/go/ql/test/go.mod deleted file mode 100644 index 2420613ecee..00000000000 --- a/go/ql/test/go.mod +++ /dev/null @@ -1,4 +0,0 @@ -module github.com/github/codeql-go/ql/test - -go 1.21 - diff --git a/go/test-runtime-version/go.work b/go/test-runtime-version/go.work new file mode 100644 index 00000000000..233b1008240 --- /dev/null +++ b/go/test-runtime-version/go.work @@ -0,0 +1 @@ +go 1.22 From 1aafc377adf9a809e59e3a66f24235e2671b1822 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 2 May 2024 16:22:24 +0200 Subject: [PATCH 31/41] Revert "Go: write test go runtime version in a specific file" This reverts commit abcd9165b4b9ded2c2b9c297502df24529f95561. --- .github/workflows/go-tests-other-os.yml | 10 ++++++---- .github/workflows/go-tests.yml | 7 +++++-- go/ql/test/go.mod | 4 ++++ go/test-runtime-version/go.work | 1 - 4 files changed, 15 insertions(+), 7 deletions(-) create mode 100644 go/ql/test/go.mod delete mode 100644 go/test-runtime-version/go.work diff --git a/.github/workflows/go-tests-other-os.yml b/.github/workflows/go-tests-other-os.yml index d38fd18f272..ded53f868b7 100644 --- a/.github/workflows/go-tests-other-os.yml +++ b/.github/workflows/go-tests-other-os.yml @@ -7,6 +7,8 @@ on: - .github/workflows/go-tests-other-os.yml - .github/actions/** - codeql-workspace.yml +env: + GO_VERSION: '~1.22.0' permissions: contents: read @@ -16,10 +18,10 @@ jobs: name: Test MacOS runs-on: macos-latest steps: - - name: Set up Go + - name: Set up Go ${{ env.GO_VERSION }} uses: actions/setup-go@v5 with: - go-version-file: go/test-runtime-version/go.work + go-version: ${{ env.GO_VERSION }} cache: false id: go @@ -53,10 +55,10 @@ jobs: name: Test Windows runs-on: windows-latest-xl steps: - - name: Set up Go + - name: Set up Go ${{ env.GO_VERSION }} uses: actions/setup-go@v5 with: - go-version-file: go/test-runtime-version/go.work + go-version: ${{ env.GO_VERSION }} cache: false id: go diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 60182bb2c81..6d9cac5dae9 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -16,6 +16,9 @@ on: - .github/actions/** - codeql-workspace.yml +env: + GO_VERSION: '~1.22.0' + permissions: contents: read @@ -25,10 +28,10 @@ jobs: name: Test Linux (Ubuntu) runs-on: ubuntu-latest-xl steps: - - name: Set up Go + - name: Set up Go ${{ env.GO_VERSION }} uses: actions/setup-go@v5 with: - go-version-file: go/test-runtime-version/go.work + go-version: ${{ env.GO_VERSION }} cache: false id: go diff --git a/go/ql/test/go.mod b/go/ql/test/go.mod new file mode 100644 index 00000000000..2420613ecee --- /dev/null +++ b/go/ql/test/go.mod @@ -0,0 +1,4 @@ +module github.com/github/codeql-go/ql/test + +go 1.21 + diff --git a/go/test-runtime-version/go.work b/go/test-runtime-version/go.work deleted file mode 100644 index 233b1008240..00000000000 --- a/go/test-runtime-version/go.work +++ /dev/null @@ -1 +0,0 @@ -go 1.22 From 76067cb12d88fdd761ac4dbfbd85eada52415977 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 2 May 2024 17:02:48 +0200 Subject: [PATCH 32/41] Go: skip `X:nocoverageredesign` printing by autobuilder built with bazel --- go/extractor/cli/go-autobuilder/go-autobuilder.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/go/extractor/cli/go-autobuilder/go-autobuilder.go b/go/extractor/cli/go-autobuilder/go-autobuilder.go index 08f8477cac7..8df8b506918 100644 --- a/go/extractor/cli/go-autobuilder/go-autobuilder.go +++ b/go/extractor/cli/go-autobuilder/go-autobuilder.go @@ -487,7 +487,9 @@ func extract(workspace project.GoWorkspace) bool { // Build the project and run the extractor. func installDependenciesAndBuild() { - log.Printf("Autobuilder was built with %s, environment has %s\n", runtime.Version(), toolchain.GetEnvGoVersion()) + // do not print experiments the autobuilder was built with if any, only the version + version := strings.SplitN(runtime.Version(), " ", 2)[0] + log.Printf("Autobuilder was built with %s, environment has %s\n", version, toolchain.GetEnvGoVersion()) srcdir := getSourceDir() From 00baccbc152af091936983ba111d0f5cc3b46c46 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 2 May 2024 17:08:23 +0200 Subject: [PATCH 33/41] Go: autoformat --- go/extractor/cli/go-autobuilder/go-autobuilder.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/extractor/cli/go-autobuilder/go-autobuilder.go b/go/extractor/cli/go-autobuilder/go-autobuilder.go index 8df8b506918..2e9731c989b 100644 --- a/go/extractor/cli/go-autobuilder/go-autobuilder.go +++ b/go/extractor/cli/go-autobuilder/go-autobuilder.go @@ -487,8 +487,8 @@ func extract(workspace project.GoWorkspace) bool { // Build the project and run the extractor. func installDependenciesAndBuild() { - // do not print experiments the autobuilder was built with if any, only the version - version := strings.SplitN(runtime.Version(), " ", 2)[0] + // do not print experiments the autobuilder was built with if any, only the version + version := strings.SplitN(runtime.Version(), " ", 2)[0] log.Printf("Autobuilder was built with %s, environment has %s\n", version, toolchain.GetEnvGoVersion()) srcdir := getSourceDir() From 31c427e64c030e0cdaeb26b7c571e05124ba00f0 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 3 May 2024 13:04:42 +0200 Subject: [PATCH 34/41] Bazel/Go: add more explanation in `gen.py` --- go/gen.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/go/gen.py b/go/gen.py index ebb57b2a685..0e9005571ea 100644 --- a/go/gen.py +++ b/go/gen.py @@ -1,3 +1,12 @@ +""" +Update generated files related to Go in the repo. Using --force will regenerate all files from scratch. + +In particular the script will: +1. update the `vendor` dir with `go work vendor` (using a go toolchain provided by bazel) +2. update `BUILD.bazel` files using gazelle +3. update `ql/lib/go.dbscheme` using a compiled `go-dbschemegen` +""" + import sys import pathlib import subprocess @@ -7,9 +16,9 @@ import shutil from python.runfiles import runfiles def options(): - p = argparse.ArgumentParser(description="Update generated checked in files in the Go pack") + p = argparse.ArgumentParser(description="Update generated files related to Go in the repo") p.add_argument("--force", "-f", action="store_true", help="Regenerate all files from scratch rather than updating them") - p.add_argument("generators", nargs=3) + p.add_argument("executables", nargs=3, help="Internally provided executables") return p.parse_args() opts = options() @@ -23,7 +32,7 @@ except KeyError: go_extractor_dir = workspace_dir / "go" / "extractor" go_dbscheme = workspace_dir / "go" / "ql" / "lib" / "go.dbscheme" r = runfiles.Create() -go, gazelle, go_gen_dbscheme = map(r.Rlocation, opts.generators) +go, gazelle, go_gen_dbscheme = map(r.Rlocation, opts.executables) if opts.force: @@ -43,13 +52,16 @@ if opts.force: print("running gazelle") subprocess.check_call([gazelle]) +# we want to stamp all newly generated `BUILD.bazel` files with a header build_files_to_update = set(go_extractor_dir.glob("*/**/BUILD.bazel")) +# if --force, all files are new if not opts.force: + # otherwise, subtract the files that existed at the start build_files_to_update -= existing_build_files - # these are always refreshed + # but bring back the `vendor` ones, as the vendor update step always clears them build_files_to_update.update(go_extractor_dir.glob("vendor/**/BUILD.bazel")) -print("adding header to generated BUILD files") +print("adding header to newly generated BUILD files") for build_file in build_files_to_update: contents = build_file.read_text() build_file.write_text(f"# generated running `bazel run //go/gazelle`, do not edit\n\n{contents}") From 8f0b88497a89ed9cce41c659528175427d61bf85 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 3 May 2024 13:07:56 +0200 Subject: [PATCH 35/41] Bazel/Go: be more specific in `go/extractor/BUILD.bazel` comments --- go/extractor/BUILD.bazel | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/go/extractor/BUILD.bazel b/go/extractor/BUILD.bazel index 32eaa8fda74..297ca52c8b8 100644 --- a/go/extractor/BUILD.bazel +++ b/go/extractor/BUILD.bazel @@ -4,7 +4,7 @@ load("@rules_pkg//pkg:mappings.bzl", "pkg_files") # gazelle:prefix github.com/github/codeql-go/extractor # gazelle:map_kind go_binary codeql_go_binary //go:rules.bzl -# following target is kept up to date by `bazel run //go:gen`, do not edit directly +# the immediately following `extractor` target is kept up to date by `bazel run //go:gen`, do not edit directly go_library( name = "extractor", srcs = [ @@ -25,6 +25,7 @@ go_library( ], ) +# notice that these other targets are not generated java_library( name = "tokenizer-deps", srcs = [ From 2132c7bf967e6975827ce8a35b3511cf8297d3f2 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 3 May 2024 14:52:17 +0200 Subject: [PATCH 36/41] Bazel/Go: make `@codeql//go:gen` runnable from internal repo --- go/BUILD.bazel | 22 +++++++--------------- go/gen.py | 12 +++++++++--- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/go/BUILD.bazel b/go/BUILD.bazel index 6c80e7cb258..e0da93475a2 100644 --- a/go/BUILD.bazel +++ b/go/BUILD.bazel @@ -1,13 +1,14 @@ load("@bazel_skylib//rules:native_binary.bzl", "native_binary") -load("@gazelle//:def.bzl", "gazelle") load("@rules_pkg//pkg:install.bzl", "pkg_install") load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files") load("@rules_pkg//pkg:zip.bzl", "pkg_zip") load("//:defs.bzl", "codeql_platform") -gazelle( +native_binary( name = "gazelle", - extra_args = ["go/extractor"], + src = "@gazelle//cmd/gazelle", + out = "gazelle.exe", + args = ["go/extractor"], ) _gen_binaries = [ @@ -24,9 +25,9 @@ py_binary( deps = ["@rules_python//python/runfiles"], ) -# this is an internal copy of the dbscheme to be used by extractor-pack -# this allows the extractor-pack target to be independent and up-to-date with respect to -# having run //go:gen to update the checked in files +# this is an instance of the dbscheme kept in the bazel build tree +# this allows everything that bazel builds to be up-to-date, +# independently from whether //go:gen was already run to update the checked in files genrule( name = "dbscheme", outs = ["go.dbscheme"], @@ -108,12 +109,3 @@ py_binary( main = "create_extractor_pack.py", deps = ["@rules_python//python/runfiles"], ) - -native_binary( - name = "gen-dbscheme", - src = "//go/extractor/cli/go-gen-dbscheme", - out = "go-gen-dbscheme", - args = [ - "$$BUILD_WORKSPACE_DIRECTORY/go/ql/lib/go.dbscheme", - ], -) diff --git a/go/gen.py b/go/gen.py index 0e9005571ea..6f8d47d7096 100644 --- a/go/gen.py +++ b/go/gen.py @@ -24,12 +24,18 @@ def options(): opts = options() try: - workspace_dir = pathlib.Path(os.environ['BUILD_WORKSPACE_DIRECTORY']) + workspace_dir = pathlib.Path(os.environ.pop('BUILD_WORKSPACE_DIRECTORY')) except KeyError: print("this should be run with bazel run", file=sys.stderr) sys.exit(1) go_extractor_dir = workspace_dir / "go" / "extractor" + +if not go_extractor_dir.exists(): + # internal repo? + workspace_dir /= "ql" + go_extractor_dir = workspace_dir / "go" / "extractor" + go_dbscheme = workspace_dir / "go" / "ql" / "lib" / "go.dbscheme" r = runfiles.Create() go, gazelle, go_gen_dbscheme = map(r.Rlocation, opts.executables) @@ -49,8 +55,8 @@ if opts.force: for build_file in existing_build_files: build_file.unlink() -print("running gazelle") -subprocess.check_call([gazelle]) +print("running gazelle", gazelle, go_extractor_dir) +subprocess.check_call([gazelle, "go/extractor"], cwd=workspace_dir) # we want to stamp all newly generated `BUILD.bazel` files with a header build_files_to_update = set(go_extractor_dir.glob("*/**/BUILD.bazel")) From 471303bd7ce7e45182cc586db60344f7bae86858 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 3 May 2024 14:56:17 +0200 Subject: [PATCH 37/41] Bazel/Go: remove unneeded comment --- MODULE.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MODULE.bazel b/MODULE.bazel index d069d320f07..27479e1978f 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -55,7 +55,7 @@ node.toolchain( use_repo(node, "nodejs", "nodejs_toolchains") go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk") -go_sdk.download(version = "1.22.2") # default +go_sdk.download(version = "1.22.2") register_toolchains( "@nodejs_toolchains//:all", From 17990da205995d0fcd7f052287ba9ecb15616e69 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 3 May 2024 15:58:43 +0200 Subject: [PATCH 38/41] Update go/extractor/BUILD.bazel Co-authored-by: Cornelius Riemenschneider --- go/extractor/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/extractor/BUILD.bazel b/go/extractor/BUILD.bazel index 297ca52c8b8..6047eea6860 100644 --- a/go/extractor/BUILD.bazel +++ b/go/extractor/BUILD.bazel @@ -25,7 +25,7 @@ go_library( ], ) -# notice that these other targets are not generated +# the other targets are not generated java_library( name = "tokenizer-deps", srcs = [ From 77128de105d35ada1e379a6dea61d3939e6cab6e Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 3 May 2024 17:44:29 +0200 Subject: [PATCH 39/41] Bazel/Go: make installer work from internal repo and on windows It turns out everything that is needed for the installer to work on windows is enabling runfiles. This also requires symlinks to avoid excessive copying of files. --- .bazelrc | 3 +++ go/BUILD.bazel | 24 +++--------------------- go/create_extractor_pack.py | 24 ++++++++++-------------- 3 files changed, 16 insertions(+), 35 deletions(-) mode change 100644 => 100755 go/create_extractor_pack.py diff --git a/.bazelrc b/.bazelrc index 0a49f682da3..36111310779 100644 --- a/.bazelrc +++ b/.bazelrc @@ -14,6 +14,9 @@ build:linux --cxxopt=-std=c++20 build:macos --cxxopt=-std=c++20 --cpu=darwin_x86_64 build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor +# this requires developer mode, but is required to have pack installer functioning +common:windows --windows_enable_symlinks --enable_runfiles + common --registry=file:///%workspace%/misc/bazel/registry common --registry=https://bcr.bazel.build diff --git a/go/BUILD.bazel b/go/BUILD.bazel index e0da93475a2..4fb73a51fb6 100644 --- a/go/BUILD.bazel +++ b/go/BUILD.bazel @@ -1,7 +1,6 @@ load("@bazel_skylib//rules:native_binary.bzl", "native_binary") load("@rules_pkg//pkg:install.bzl", "pkg_install") load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files") -load("@rules_pkg//pkg:zip.bzl", "pkg_zip") load("//:defs.bzl", "codeql_platform") native_binary( @@ -81,31 +80,14 @@ pkg_filegroup( ) pkg_install( - name = "_extractor-pack-installer", + name = "_extractor_pack", srcs = [":extractor-pack"], ) -# rules_pkg installer is currently broken on Windows -# see https://github.com/bazelbuild/rules_pkg/issues/387 -# for now, work around it using an archive -pkg_zip( - name = "_extractor-pack-zip", - srcs = [":extractor-pack"], -) - -alias( - name = "_create-extractor-pack-arg", - actual = select({ - "@platforms//os:windows": ":_extractor-pack-zip", - "//conditions:default": ":_extractor-pack-installer", - }), -) - py_binary( name = "create-extractor-pack", srcs = ["create_extractor_pack.py"], - args = ["$(rlocationpath :_create-extractor-pack-arg)"], - data = [":_create-extractor-pack-arg"], + env = {"REPO_NAME": repo_name()}, main = "create_extractor_pack.py", - deps = ["@rules_python//python/runfiles"], + deps = ["_extractor_pack"], ) diff --git a/go/create_extractor_pack.py b/go/create_extractor_pack.py old mode 100644 new mode 100755 index a1154a777d8..4d194ab93e4 --- a/go/create_extractor_pack.py +++ b/go/create_extractor_pack.py @@ -1,26 +1,22 @@ +#!/usr/bin/env python3 import os import pathlib import shutil import sys import subprocess -import zipfile -from python.runfiles import runfiles try: workspace_dir = pathlib.Path(os.environ['BUILD_WORKSPACE_DIRECTORY']) except KeyError: - print("this should be run with bazel run", file=sys.stderr) - sys.exit(1) + res = subprocess.run(["bazel", "run", ":create-extractor-pack"], cwd=pathlib.Path(__file__).parent) + sys.exit(res.returncode) -dest_dir = workspace_dir / 'go' / 'build' / 'codeql-extractor-go' -installer_or_zip = pathlib.Path(runfiles.Create().Rlocation(sys.argv[1])) +from go._extractor_pack_install_script import main +if os.environ['REPO_NAME'] == 'codeql~': + workspace_dir /= 'ql' + +dest_dir = workspace_dir / 'go' / 'build' / 'codeql-extractor-pack' shutil.rmtree(dest_dir, ignore_errors=True) - -if installer_or_zip.suffix == '.zip': - dest_dir.mkdir() - with zipfile.ZipFile(installer_or_zip) as pack: - pack.extractall(dest_dir) -else: - os.environ['DESTDIR'] = str(dest_dir) - subprocess.check_call([installer_or_zip]) +os.environ['DESTDIR'] = str(dest_dir) +main(sys.argv) From 5b184c179adfcc4a5f3accf86cb617ebdf9a7275 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 6 May 2024 12:47:51 +0200 Subject: [PATCH 40/41] Bazel/Go: add some comments --- go/extractor/BUILD.bazel | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/go/extractor/BUILD.bazel b/go/extractor/BUILD.bazel index 6047eea6860..7e576927f66 100644 --- a/go/extractor/BUILD.bazel +++ b/go/extractor/BUILD.bazel @@ -25,7 +25,9 @@ go_library( ], ) -# the other targets are not generated +# the other targets are not generated by gazelle + +# this is separate from `tokenizer-jar` below because we don't want these compiled class files in the pack java_library( name = "tokenizer-deps", srcs = [ @@ -36,6 +38,7 @@ java_library( ], ) +# we only need these compiled class files in the pack java_library( name = "tokenizer-jar", srcs = [ @@ -51,7 +54,7 @@ pkg_files( srcs = [":tokenizer-jar"], prefix = "tools", renames = { - ":tokenizer-jar": "tokenizer.jar", + ":tokenizer-jar": "tokenizer.jar", # name is `libtokenizer.jar` by default }, visibility = ["//go:__pkg__"], ) From 73df4fa920bfdef11cd8ecb094092f103bfa4850 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 6 May 2024 14:55:35 +0200 Subject: [PATCH 41/41] Go: fix Windows installation --- .bazelrc | 3 ++- go/create_extractor_pack.py | 10 +++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.bazelrc b/.bazelrc index 36111310779..c2b4d3b7f03 100644 --- a/.bazelrc +++ b/.bazelrc @@ -15,7 +15,8 @@ build:macos --cxxopt=-std=c++20 --cpu=darwin_x86_64 build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor # this requires developer mode, but is required to have pack installer functioning -common:windows --windows_enable_symlinks --enable_runfiles +startup --windows_enable_symlinks +common --enable_runfiles common --registry=file:///%workspace%/misc/bazel/registry common --registry=https://bcr.bazel.build diff --git a/go/create_extractor_pack.py b/go/create_extractor_pack.py index 4d194ab93e4..427f0c37c36 100755 --- a/go/create_extractor_pack.py +++ b/go/create_extractor_pack.py @@ -13,10 +13,14 @@ except KeyError: from go._extractor_pack_install_script import main -if os.environ['REPO_NAME'] == 'codeql~': - workspace_dir /= 'ql' +build_dir = workspace_dir / 'go' / 'build' -dest_dir = workspace_dir / 'go' / 'build' / 'codeql-extractor-pack' +if not build_dir.exists(): + # we probably are in the internal repo + workspace_dir /= 'ql' + build_dir = workspace_dir / 'go' / 'build' + +dest_dir = build_dir / 'codeql-extractor-pack' shutil.rmtree(dest_dir, ignore_errors=True) os.environ['DESTDIR'] = str(dest_dir) main(sys.argv)