diff --git a/.bazelrc b/.bazelrc index 0a49f682da3..c2b4d3b7f03 100644 --- a/.bazelrc +++ b/.bazelrc @@ -14,6 +14,10 @@ build:linux --cxxopt=-std=c++20 build:macos --cxxopt=-std=c++20 --cpu=darwin_x86_64 build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor +# this requires developer mode, but is required to have pack installer functioning +startup --windows_enable_symlinks +common --enable_runfiles + common --registry=file:///%workspace%/misc/bazel/registry common --registry=https://bcr.bazel.build diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 00c34e5df06..05104490468 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,6 +29,13 @@ repos: entry: bazel run //misc/bazel:buildifier pass_filenames: false + - id: go-gen + name: Check checked in generated files in go + files: ^go/.* + language: system + entry: bazel run //go:gen + pass_filenames: false + - id: codeql-format name: Fix QL file formatting files: \.qll?$ diff --git a/MODULE.bazel b/MODULE.bazel index c834acab7ae..27479e1978f 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -13,7 +13,8 @@ local_path_override( # see https://registry.bazel.build/ for a list of available packages -bazel_dep(name = "platforms", version = "0.0.8") +bazel_dep(name = "platforms", version = "0.0.9") +bazel_dep(name = "rules_go", version = "0.47.0") bazel_dep(name = "rules_pkg", version = "0.10.1") bazel_dep(name = "rules_nodejs", version = "6.0.3") bazel_dep(name = "rules_python", version = "0.31.0") @@ -21,6 +22,7 @@ bazel_dep(name = "bazel_skylib", version = "1.5.0") bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "absl") bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "json") bazel_dep(name = "fmt", version = "10.0.0") +bazel_dep(name = "gazelle", version = "0.36.0") bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True) @@ -52,6 +54,9 @@ node.toolchain( ) use_repo(node, "nodejs", "nodejs_toolchains") +go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk") +go_sdk.download(version = "1.22.2") + register_toolchains( "@nodejs_toolchains//:all", ) diff --git a/go/BUILD.bazel b/go/BUILD.bazel new file mode 100644 index 00000000000..4fb73a51fb6 --- /dev/null +++ b/go/BUILD.bazel @@ -0,0 +1,93 @@ +load("@bazel_skylib//rules:native_binary.bzl", "native_binary") +load("@rules_pkg//pkg:install.bzl", "pkg_install") +load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files") +load("//:defs.bzl", "codeql_platform") + +native_binary( + name = "gazelle", + src = "@gazelle//cmd/gazelle", + out = "gazelle.exe", + args = ["go/extractor"], +) + +_gen_binaries = [ + "@rules_go//go", + ":gazelle", + "//go/extractor/cli/go-gen-dbscheme", +] + +py_binary( + name = "gen", + srcs = ["gen.py"], + args = ["$(rlocationpath %s)" % bin for bin in _gen_binaries], + data = _gen_binaries, + deps = ["@rules_python//python/runfiles"], +) + +# this is an instance of the dbscheme kept in the bazel build tree +# this allows everything that bazel builds to be up-to-date, +# independently from whether //go:gen was already run to update the checked in files +genrule( + name = "dbscheme", + outs = ["go.dbscheme"], + cmd = "$(execpath //go/extractor/cli/go-gen-dbscheme) $@", + tools = ["//go/extractor/cli/go-gen-dbscheme"], +) + +pkg_files( + name = "resources", + srcs = [ + "LICENSE", + "codeql-extractor.yml", + "ql/lib/go.dbscheme.stats", + ":dbscheme", + ], +) + +pkg_filegroup( + name = "extractor-pack-generic", + srcs = [ + ":resources", + "//go/codeql-tools", + "//go/downgrades", + "//go/extractor:tokenizer", + ], + visibility = ["//visibility:public"], +) + +pkg_files( + name = "extractor-pack-arch", + srcs = [ + "//go/extractor/cli/go-autobuilder", + "//go/extractor/cli/go-bootstrap", + "//go/extractor/cli/go-build-runner", + "//go/extractor/cli/go-extractor", + "//go/extractor/cli/go-gen-dbscheme", + "//go/extractor/cli/go-tokenizer", + ], + attributes = pkg_attributes(mode = "0755"), + prefix = "tools/" + codeql_platform, + visibility = ["//visibility:public"], +) + +pkg_filegroup( + name = "extractor-pack", + srcs = [ + ":extractor-pack-arch", + ":extractor-pack-generic", + ], + visibility = ["//visibility:public"], +) + +pkg_install( + name = "_extractor_pack", + srcs = [":extractor-pack"], +) + +py_binary( + name = "create-extractor-pack", + srcs = ["create_extractor_pack.py"], + env = {"REPO_NAME": repo_name()}, + main = "create_extractor_pack.py", + deps = ["_extractor_pack"], +) diff --git a/go/codeql-tools/BUILD.bazel b/go/codeql-tools/BUILD.bazel new file mode 100644 index 00000000000..8c3946b24ad --- /dev/null +++ b/go/codeql-tools/BUILD.bazel @@ -0,0 +1,28 @@ +load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files") + +pkg_files( + name = "sh-files", + srcs = glob(["*.sh"]), + attributes = pkg_attributes(mode = "0755"), +) + +pkg_files( + name = "non-sh-files", + srcs = glob( + ["*"], + exclude = [ + "*.sh", + "BUILD.bazel", + ], + ), +) + +pkg_filegroup( + name = "codeql-tools", + srcs = [ + ":non-sh-files", + ":sh-files", + ], + prefix = "tools", + visibility = ["//go:__pkg__"], +) diff --git a/go/create_extractor_pack.py b/go/create_extractor_pack.py new file mode 100755 index 00000000000..427f0c37c36 --- /dev/null +++ b/go/create_extractor_pack.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +import os +import pathlib +import shutil +import sys +import subprocess + +try: + workspace_dir = pathlib.Path(os.environ['BUILD_WORKSPACE_DIRECTORY']) +except KeyError: + res = subprocess.run(["bazel", "run", ":create-extractor-pack"], cwd=pathlib.Path(__file__).parent) + sys.exit(res.returncode) + +from go._extractor_pack_install_script import main + +build_dir = workspace_dir / 'go' / 'build' + +if not build_dir.exists(): + # we probably are in the internal repo + workspace_dir /= 'ql' + build_dir = workspace_dir / 'go' / 'build' + +dest_dir = build_dir / 'codeql-extractor-pack' +shutil.rmtree(dest_dir, ignore_errors=True) +os.environ['DESTDIR'] = str(dest_dir) +main(sys.argv) diff --git a/go/downgrades/BUILD.bazel b/go/downgrades/BUILD.bazel new file mode 100644 index 00000000000..68c15741a9f --- /dev/null +++ b/go/downgrades/BUILD.bazel @@ -0,0 +1,12 @@ +load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "strip_prefix") + +pkg_files( + name = "downgrades", + srcs = glob( + ["**"], + exclude = ["BUILD.bazel"], + ), + prefix = "downgrades", + strip_prefix = strip_prefix.from_pkg(), + visibility = ["//go:__pkg__"], +) diff --git a/go/extractor/.gitattributes b/go/extractor/.gitattributes new file mode 100644 index 00000000000..e406d413677 --- /dev/null +++ b/go/extractor/.gitattributes @@ -0,0 +1 @@ +/*/**/BUILD.bazel linguist-generated=true diff --git a/go/extractor/BUILD.bazel b/go/extractor/BUILD.bazel new file mode 100644 index 00000000000..7e576927f66 --- /dev/null +++ b/go/extractor/BUILD.bazel @@ -0,0 +1,60 @@ +load("@rules_go//go:def.bzl", "go_library") +load("@rules_pkg//pkg:mappings.bzl", "pkg_files") + +# gazelle:prefix github.com/github/codeql-go/extractor +# gazelle:map_kind go_binary codeql_go_binary //go:rules.bzl + +# the immediately following `extractor` target is kept up to date by `bazel run //go:gen`, do not edit directly +go_library( + name = "extractor", + srcs = [ + "extractor.go", + "gomodextractor.go", + "semaphore.go", + ], + importpath = "github.com/github/codeql-go/extractor", + visibility = ["//visibility:public"], + deps = [ + "//go/extractor/dbscheme", + "//go/extractor/diagnostics", + "//go/extractor/srcarchive", + "//go/extractor/trap", + "//go/extractor/util", + "//go/extractor/vendor/golang.org/x/mod/modfile", + "//go/extractor/vendor/golang.org/x/tools/go/packages", + ], +) + +# the other targets are not generated by gazelle + +# this is separate from `tokenizer-jar` below because we don't want these compiled class files in the pack +java_library( + name = "tokenizer-deps", + srcs = [ + "net/sourceforge/pmd/cpd/AbstractLanguage.java", + "net/sourceforge/pmd/cpd/SourceCode.java", + "net/sourceforge/pmd/cpd/TokenEntry.java", + "net/sourceforge/pmd/cpd/Tokenizer.java", + ], +) + +# we only need these compiled class files in the pack +java_library( + name = "tokenizer-jar", + srcs = [ + "net/sourceforge/pmd/cpd/GoLanguage.java", + "opencsv/CSVParser.java", + "opencsv/CSVReader.java", + ], + deps = [":tokenizer-deps"], +) + +pkg_files( + name = "tokenizer", + srcs = [":tokenizer-jar"], + prefix = "tools", + renames = { + ":tokenizer-jar": "tokenizer.jar", # name is `libtokenizer.jar` by default + }, + visibility = ["//go:__pkg__"], +) diff --git a/go/extractor/autobuilder/BUILD.bazel b/go/extractor/autobuilder/BUILD.bazel new file mode 100644 index 00000000000..e40dc3a0321 --- /dev/null +++ b/go/extractor/autobuilder/BUILD.bazel @@ -0,0 +1,26 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "autobuilder", + srcs = [ + "autobuilder.go", + "build-environment.go", + ], + importpath = "github.com/github/codeql-go/extractor/autobuilder", + visibility = ["//visibility:public"], + deps = [ + "//go/extractor/diagnostics", + "//go/extractor/project", + "//go/extractor/toolchain", + "//go/extractor/util", + "//go/extractor/vendor/golang.org/x/mod/semver", + ], +) + +go_test( + name = "autobuilder_test", + srcs = ["build-environment_test.go"], + embed = [":autobuilder"], +) diff --git a/go/extractor/cli/go-autobuilder/BUILD.bazel b/go/extractor/cli/go-autobuilder/BUILD.bazel new file mode 100644 index 00000000000..7abf4600d94 --- /dev/null +++ b/go/extractor/cli/go-autobuilder/BUILD.bazel @@ -0,0 +1,25 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library") +load("//go:rules.bzl", "codeql_go_binary") + +go_library( + name = "go-autobuilder_lib", + srcs = ["go-autobuilder.go"], + importpath = "github.com/github/codeql-go/extractor/cli/go-autobuilder", + visibility = ["//visibility:private"], + deps = [ + "//go/extractor/autobuilder", + "//go/extractor/diagnostics", + "//go/extractor/project", + "//go/extractor/toolchain", + "//go/extractor/util", + "//go/extractor/vendor/golang.org/x/mod/semver", + ], +) + +codeql_go_binary( + name = "go-autobuilder", + embed = [":go-autobuilder_lib"], + visibility = ["//visibility:public"], +) diff --git a/go/extractor/cli/go-autobuilder/go-autobuilder.go b/go/extractor/cli/go-autobuilder/go-autobuilder.go index 08f8477cac7..2e9731c989b 100644 --- a/go/extractor/cli/go-autobuilder/go-autobuilder.go +++ b/go/extractor/cli/go-autobuilder/go-autobuilder.go @@ -487,7 +487,9 @@ func extract(workspace project.GoWorkspace) bool { // Build the project and run the extractor. func installDependenciesAndBuild() { - log.Printf("Autobuilder was built with %s, environment has %s\n", runtime.Version(), toolchain.GetEnvGoVersion()) + // do not print experiments the autobuilder was built with if any, only the version + version := strings.SplitN(runtime.Version(), " ", 2)[0] + log.Printf("Autobuilder was built with %s, environment has %s\n", version, toolchain.GetEnvGoVersion()) srcdir := getSourceDir() diff --git a/go/extractor/cli/go-bootstrap/BUILD.bazel b/go/extractor/cli/go-bootstrap/BUILD.bazel new file mode 100644 index 00000000000..86b08dfa121 --- /dev/null +++ b/go/extractor/cli/go-bootstrap/BUILD.bazel @@ -0,0 +1,17 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library") +load("//go:rules.bzl", "codeql_go_binary") + +go_library( + name = "go-bootstrap_lib", + srcs = ["go-bootstrap.go"], + importpath = "github.com/github/codeql-go/extractor/cli/go-bootstrap", + visibility = ["//visibility:private"], +) + +codeql_go_binary( + name = "go-bootstrap", + embed = [":go-bootstrap_lib"], + visibility = ["//visibility:public"], +) diff --git a/go/extractor/cli/go-build-runner/BUILD.bazel b/go/extractor/cli/go-build-runner/BUILD.bazel new file mode 100644 index 00000000000..e91c4530610 --- /dev/null +++ b/go/extractor/cli/go-build-runner/BUILD.bazel @@ -0,0 +1,21 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library") +load("//go:rules.bzl", "codeql_go_binary") + +go_library( + name = "go-build-runner_lib", + srcs = ["go-build-runner.go"], + importpath = "github.com/github/codeql-go/extractor/cli/go-build-runner", + visibility = ["//visibility:private"], + deps = [ + "//go/extractor/autobuilder", + "//go/extractor/util", + ], +) + +codeql_go_binary( + name = "go-build-runner", + embed = [":go-build-runner_lib"], + visibility = ["//visibility:public"], +) diff --git a/go/extractor/cli/go-extractor/BUILD.bazel b/go/extractor/cli/go-extractor/BUILD.bazel new file mode 100644 index 00000000000..769e4a7b09b --- /dev/null +++ b/go/extractor/cli/go-extractor/BUILD.bazel @@ -0,0 +1,21 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library") +load("//go:rules.bzl", "codeql_go_binary") + +go_library( + name = "go-extractor_lib", + srcs = ["go-extractor.go"], + importpath = "github.com/github/codeql-go/extractor/cli/go-extractor", + visibility = ["//visibility:private"], + deps = [ + "//go/extractor", + "//go/extractor/diagnostics", + ], +) + +codeql_go_binary( + name = "go-extractor", + embed = [":go-extractor_lib"], + visibility = ["//visibility:public"], +) diff --git a/go/extractor/cli/go-gen-dbscheme/BUILD.bazel b/go/extractor/cli/go-gen-dbscheme/BUILD.bazel new file mode 100644 index 00000000000..9de6d2198f8 --- /dev/null +++ b/go/extractor/cli/go-gen-dbscheme/BUILD.bazel @@ -0,0 +1,18 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library") +load("//go:rules.bzl", "codeql_go_binary") + +go_library( + name = "go-gen-dbscheme_lib", + srcs = ["go-gen-dbscheme.go"], + importpath = "github.com/github/codeql-go/extractor/cli/go-gen-dbscheme", + visibility = ["//visibility:private"], + deps = ["//go/extractor/dbscheme"], +) + +codeql_go_binary( + name = "go-gen-dbscheme", + embed = [":go-gen-dbscheme_lib"], + visibility = ["//visibility:public"], +) diff --git a/go/extractor/cli/go-tokenizer/BUILD.bazel b/go/extractor/cli/go-tokenizer/BUILD.bazel new file mode 100644 index 00000000000..8ce0c76be26 --- /dev/null +++ b/go/extractor/cli/go-tokenizer/BUILD.bazel @@ -0,0 +1,17 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library") +load("//go:rules.bzl", "codeql_go_binary") + +go_library( + name = "go-tokenizer_lib", + srcs = ["go-tokenizer.go"], + importpath = "github.com/github/codeql-go/extractor/cli/go-tokenizer", + visibility = ["//visibility:private"], +) + +codeql_go_binary( + name = "go-tokenizer", + embed = [":go-tokenizer_lib"], + visibility = ["//visibility:public"], +) diff --git a/go/extractor/dbscheme/BUILD.bazel b/go/extractor/dbscheme/BUILD.bazel new file mode 100644 index 00000000000..496a5ccdf49 --- /dev/null +++ b/go/extractor/dbscheme/BUILD.bazel @@ -0,0 +1,17 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "dbscheme", + srcs = [ + "dbscheme.go", + "tables.go", + ], + importpath = "github.com/github/codeql-go/extractor/dbscheme", + visibility = ["//visibility:public"], + deps = [ + "//go/extractor/trap", + "//go/extractor/vendor/golang.org/x/tools/go/packages", + ], +) diff --git a/go/extractor/diagnostics/BUILD.bazel b/go/extractor/diagnostics/BUILD.bazel new file mode 100644 index 00000000000..9b0c148db32 --- /dev/null +++ b/go/extractor/diagnostics/BUILD.bazel @@ -0,0 +1,10 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "diagnostics", + srcs = ["diagnostics.go"], + importpath = "github.com/github/codeql-go/extractor/diagnostics", + visibility = ["//visibility:public"], +) diff --git a/go/extractor/project/BUILD.bazel b/go/extractor/project/BUILD.bazel new file mode 100644 index 00000000000..fa7ee19875d --- /dev/null +++ b/go/extractor/project/BUILD.bazel @@ -0,0 +1,24 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "project", + srcs = ["project.go"], + importpath = "github.com/github/codeql-go/extractor/project", + visibility = ["//visibility:public"], + deps = [ + "//go/extractor/diagnostics", + "//go/extractor/toolchain", + "//go/extractor/util", + "//go/extractor/vendor/golang.org/x/mod/modfile", + "//go/extractor/vendor/golang.org/x/mod/semver", + ], +) + +go_test( + name = "project_test", + srcs = ["project_test.go"], + embed = [":project"], + deps = ["//go/extractor/vendor/golang.org/x/mod/modfile"], +) diff --git a/go/extractor/srcarchive/BUILD.bazel b/go/extractor/srcarchive/BUILD.bazel new file mode 100644 index 00000000000..e72e2e7ca08 --- /dev/null +++ b/go/extractor/srcarchive/BUILD.bazel @@ -0,0 +1,19 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "srcarchive", + srcs = [ + "projectlayout.go", + "srcarchive.go", + ], + importpath = "github.com/github/codeql-go/extractor/srcarchive", + visibility = ["//visibility:public"], +) + +go_test( + name = "srcarchive_test", + srcs = ["projectlayout_test.go"], + embed = [":srcarchive"], +) diff --git a/go/extractor/toolchain/BUILD.bazel b/go/extractor/toolchain/BUILD.bazel new file mode 100644 index 00000000000..fde8d327e9e --- /dev/null +++ b/go/extractor/toolchain/BUILD.bazel @@ -0,0 +1,20 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "toolchain", + srcs = ["toolchain.go"], + importpath = "github.com/github/codeql-go/extractor/toolchain", + visibility = ["//visibility:public"], + deps = [ + "//go/extractor/util", + "//go/extractor/vendor/golang.org/x/mod/semver", + ], +) + +go_test( + name = "toolchain_test", + srcs = ["toolchain_test.go"], + embed = [":toolchain"], +) diff --git a/go/extractor/trap/BUILD.bazel b/go/extractor/trap/BUILD.bazel new file mode 100644 index 00000000000..6cc7c4983b2 --- /dev/null +++ b/go/extractor/trap/BUILD.bazel @@ -0,0 +1,25 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "trap", + srcs = [ + "labels.go", + "trapwriter.go", + "util.go", + ], + importpath = "github.com/github/codeql-go/extractor/trap", + visibility = ["//visibility:public"], + deps = [ + "//go/extractor/srcarchive", + "//go/extractor/util", + "//go/extractor/vendor/golang.org/x/tools/go/packages", + ], +) + +go_test( + name = "trap_test", + srcs = ["trapwriter_test.go"], + embed = [":trap"], +) diff --git a/go/extractor/util/BUILD.bazel b/go/extractor/util/BUILD.bazel new file mode 100644 index 00000000000..8b8869cac52 --- /dev/null +++ b/go/extractor/util/BUILD.bazel @@ -0,0 +1,16 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "util", + srcs = ["util.go"], + importpath = "github.com/github/codeql-go/extractor/util", + visibility = ["//visibility:public"], +) + +go_test( + name = "util_test", + srcs = ["util_test.go"], + embed = [":util"], +) diff --git a/go/extractor/vendor/golang.org/x/mod/internal/lazyregexp/BUILD.bazel b/go/extractor/vendor/golang.org/x/mod/internal/lazyregexp/BUILD.bazel new file mode 100644 index 00000000000..deb5dc2b019 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/mod/internal/lazyregexp/BUILD.bazel @@ -0,0 +1,11 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "lazyregexp", + srcs = ["lazyre.go"], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/mod/internal/lazyregexp", + importpath = "golang.org/x/mod/internal/lazyregexp", + visibility = ["//go/extractor/vendor/golang.org/x/mod:__subpackages__"], +) diff --git a/go/extractor/vendor/golang.org/x/mod/modfile/BUILD.bazel b/go/extractor/vendor/golang.org/x/mod/modfile/BUILD.bazel new file mode 100644 index 00000000000..097bacb107c --- /dev/null +++ b/go/extractor/vendor/golang.org/x/mod/modfile/BUILD.bazel @@ -0,0 +1,21 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "modfile", + srcs = [ + "print.go", + "read.go", + "rule.go", + "work.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/mod/modfile", + importpath = "golang.org/x/mod/modfile", + visibility = ["//visibility:public"], + deps = [ + "//go/extractor/vendor/golang.org/x/mod/internal/lazyregexp", + "//go/extractor/vendor/golang.org/x/mod/module", + "//go/extractor/vendor/golang.org/x/mod/semver", + ], +) diff --git a/go/extractor/vendor/golang.org/x/mod/module/BUILD.bazel b/go/extractor/vendor/golang.org/x/mod/module/BUILD.bazel new file mode 100644 index 00000000000..3bf5ae9997d --- /dev/null +++ b/go/extractor/vendor/golang.org/x/mod/module/BUILD.bazel @@ -0,0 +1,18 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "module", + srcs = [ + "module.go", + "pseudo.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/mod/module", + importpath = "golang.org/x/mod/module", + visibility = ["//visibility:public"], + deps = [ + "//go/extractor/vendor/golang.org/x/mod/internal/lazyregexp", + "//go/extractor/vendor/golang.org/x/mod/semver", + ], +) diff --git a/go/extractor/vendor/golang.org/x/mod/semver/BUILD.bazel b/go/extractor/vendor/golang.org/x/mod/semver/BUILD.bazel new file mode 100644 index 00000000000..760be56c9e0 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/mod/semver/BUILD.bazel @@ -0,0 +1,11 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "semver", + srcs = ["semver.go"], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/mod/semver", + importpath = "golang.org/x/mod/semver", + visibility = ["//visibility:public"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/go/gcexportdata/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/go/gcexportdata/BUILD.bazel new file mode 100644 index 00000000000..5d68c2fe989 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/go/gcexportdata/BUILD.bazel @@ -0,0 +1,15 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "gcexportdata", + srcs = [ + "gcexportdata.go", + "importer.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/go/gcexportdata", + importpath = "golang.org/x/tools/go/gcexportdata", + visibility = ["//visibility:public"], + deps = ["//go/extractor/vendor/golang.org/x/tools/internal/gcimporter"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/go/internal/packagesdriver/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/go/internal/packagesdriver/BUILD.bazel new file mode 100644 index 00000000000..2ef27e2c88a --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/go/internal/packagesdriver/BUILD.bazel @@ -0,0 +1,12 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "packagesdriver", + srcs = ["sizes.go"], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/go/internal/packagesdriver", + importpath = "golang.org/x/tools/go/internal/packagesdriver", + visibility = ["//go/extractor/vendor/golang.org/x/tools/go:__subpackages__"], + deps = ["//go/extractor/vendor/golang.org/x/tools/internal/gocommand"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/go/packages/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/go/packages/BUILD.bazel new file mode 100644 index 00000000000..03d3e3b0158 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/go/packages/BUILD.bazel @@ -0,0 +1,27 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "packages", + srcs = [ + "doc.go", + "external.go", + "golist.go", + "golist_overlay.go", + "loadmode_string.go", + "packages.go", + "visit.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/go/packages", + importpath = "golang.org/x/tools/go/packages", + visibility = ["//visibility:public"], + deps = [ + "//go/extractor/vendor/golang.org/x/tools/go/gcexportdata", + "//go/extractor/vendor/golang.org/x/tools/go/internal/packagesdriver", + "//go/extractor/vendor/golang.org/x/tools/internal/gocommand", + "//go/extractor/vendor/golang.org/x/tools/internal/packagesinternal", + "//go/extractor/vendor/golang.org/x/tools/internal/typesinternal", + "//go/extractor/vendor/golang.org/x/tools/internal/versions", + ], +) diff --git a/go/extractor/vendor/golang.org/x/tools/go/types/objectpath/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/go/types/objectpath/BUILD.bazel new file mode 100644 index 00000000000..374c5c601bc --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/go/types/objectpath/BUILD.bazel @@ -0,0 +1,12 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "objectpath", + srcs = ["objectpath.go"], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/go/types/objectpath", + importpath = "golang.org/x/tools/go/types/objectpath", + visibility = ["//visibility:public"], + deps = ["//go/extractor/vendor/golang.org/x/tools/internal/typeparams"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/event/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/event/BUILD.bazel new file mode 100644 index 00000000000..200e436fcd4 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/event/BUILD.bazel @@ -0,0 +1,19 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "event", + srcs = [ + "doc.go", + "event.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/event", + importpath = "golang.org/x/tools/internal/event", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], + deps = [ + "//go/extractor/vendor/golang.org/x/tools/internal/event/core", + "//go/extractor/vendor/golang.org/x/tools/internal/event/keys", + "//go/extractor/vendor/golang.org/x/tools/internal/event/label", + ], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/event/core/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/event/core/BUILD.bazel new file mode 100644 index 00000000000..a16713f536c --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/event/core/BUILD.bazel @@ -0,0 +1,19 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "core", + srcs = [ + "event.go", + "export.go", + "fast.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/event/core", + importpath = "golang.org/x/tools/internal/event/core", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], + deps = [ + "//go/extractor/vendor/golang.org/x/tools/internal/event/keys", + "//go/extractor/vendor/golang.org/x/tools/internal/event/label", + ], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/event/keys/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/event/keys/BUILD.bazel new file mode 100644 index 00000000000..1feefdf1a83 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/event/keys/BUILD.bazel @@ -0,0 +1,16 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "keys", + srcs = [ + "keys.go", + "standard.go", + "util.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/event/keys", + importpath = "golang.org/x/tools/internal/event/keys", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], + deps = ["//go/extractor/vendor/golang.org/x/tools/internal/event/label"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/event/label/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/event/label/BUILD.bazel new file mode 100644 index 00000000000..a4430ba0a17 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/event/label/BUILD.bazel @@ -0,0 +1,11 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "label", + srcs = ["label.go"], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/event/label", + importpath = "golang.org/x/tools/internal/event/label", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/event/tag/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/event/tag/BUILD.bazel new file mode 100644 index 00000000000..d2c87f41a8a --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/event/tag/BUILD.bazel @@ -0,0 +1,12 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "tag", + srcs = ["tag.go"], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/event/tag", + importpath = "golang.org/x/tools/internal/event/tag", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], + deps = ["//go/extractor/vendor/golang.org/x/tools/internal/event/keys"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/gcimporter/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/gcimporter/BUILD.bazel new file mode 100644 index 00000000000..56da3b0130e --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/gcimporter/BUILD.bazel @@ -0,0 +1,29 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "gcimporter", + srcs = [ + "bimport.go", + "exportdata.go", + "gcimporter.go", + "iexport.go", + "iimport.go", + "newInterface10.go", + "newInterface11.go", + "support_go117.go", + "support_go118.go", + "unified_no.go", + "ureader_no.go", + "ureader_yes.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/gcimporter", + importpath = "golang.org/x/tools/internal/gcimporter", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], + deps = [ + "//go/extractor/vendor/golang.org/x/tools/go/types/objectpath", + "//go/extractor/vendor/golang.org/x/tools/internal/pkgbits", + "//go/extractor/vendor/golang.org/x/tools/internal/tokeninternal", + ], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/gocommand/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/gocommand/BUILD.bazel new file mode 100644 index 00000000000..7e64f94b95c --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/gocommand/BUILD.bazel @@ -0,0 +1,22 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "gocommand", + srcs = [ + "invoke.go", + "vendor.go", + "version.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/gocommand", + importpath = "golang.org/x/tools/internal/gocommand", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], + deps = [ + "//go/extractor/vendor/golang.org/x/mod/semver", + "//go/extractor/vendor/golang.org/x/tools/internal/event", + "//go/extractor/vendor/golang.org/x/tools/internal/event/keys", + "//go/extractor/vendor/golang.org/x/tools/internal/event/label", + "//go/extractor/vendor/golang.org/x/tools/internal/event/tag", + ], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/packagesinternal/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/packagesinternal/BUILD.bazel new file mode 100644 index 00000000000..2d2b7dc5b33 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/packagesinternal/BUILD.bazel @@ -0,0 +1,11 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "packagesinternal", + srcs = ["packages.go"], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/packagesinternal", + importpath = "golang.org/x/tools/internal/packagesinternal", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/pkgbits/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/pkgbits/BUILD.bazel new file mode 100644 index 00000000000..cce32747051 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/pkgbits/BUILD.bazel @@ -0,0 +1,23 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "pkgbits", + srcs = [ + "codes.go", + "decoder.go", + "doc.go", + "encoder.go", + "flags.go", + "frames_go1.go", + "frames_go17.go", + "reloc.go", + "support.go", + "sync.go", + "syncmarker_string.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/pkgbits", + importpath = "golang.org/x/tools/internal/pkgbits", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/tokeninternal/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/tokeninternal/BUILD.bazel new file mode 100644 index 00000000000..c0f6cc8fb13 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/tokeninternal/BUILD.bazel @@ -0,0 +1,11 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "tokeninternal", + srcs = ["tokeninternal.go"], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/tokeninternal", + importpath = "golang.org/x/tools/internal/tokeninternal", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/typeparams/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/typeparams/BUILD.bazel new file mode 100644 index 00000000000..9c2dc20b6c6 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/typeparams/BUILD.bazel @@ -0,0 +1,17 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "typeparams", + srcs = [ + "common.go", + "coretype.go", + "normalize.go", + "termlist.go", + "typeterm.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/typeparams", + importpath = "golang.org/x/tools/internal/typeparams", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/typesinternal/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/typesinternal/BUILD.bazel new file mode 100644 index 00000000000..653752ab715 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/typesinternal/BUILD.bazel @@ -0,0 +1,16 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "typesinternal", + srcs = [ + "errorcode.go", + "errorcode_string.go", + "types.go", + "types_118.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/typesinternal", + importpath = "golang.org/x/tools/internal/typesinternal", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/versions/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/versions/BUILD.bazel new file mode 100644 index 00000000000..85d428debf5 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/versions/BUILD.bazel @@ -0,0 +1,17 @@ +# generated running `bazel run //go/gazelle`, do not edit + +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "versions", + srcs = [ + "gover.go", + "types.go", + "types_go121.go", + "types_go122.go", + "versions.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/versions", + importpath = "golang.org/x/tools/internal/versions", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], +) diff --git a/go/extractor/vendor/modules.txt b/go/extractor/vendor/modules.txt index 5687615f62c..37c4a68d828 100644 --- a/go/extractor/vendor/modules.txt +++ b/go/extractor/vendor/modules.txt @@ -1,3 +1,4 @@ +## workspace # golang.org/x/mod v0.15.0 ## explicit; go 1.18 golang.org/x/mod/internal/lazyregexp diff --git a/go/gen.py b/go/gen.py new file mode 100644 index 00000000000..6f8d47d7096 --- /dev/null +++ b/go/gen.py @@ -0,0 +1,75 @@ +""" +Update generated files related to Go in the repo. Using --force will regenerate all files from scratch. + +In particular the script will: +1. update the `vendor` dir with `go work vendor` (using a go toolchain provided by bazel) +2. update `BUILD.bazel` files using gazelle +3. update `ql/lib/go.dbscheme` using a compiled `go-dbschemegen` +""" + +import sys +import pathlib +import subprocess +import os +import argparse +import shutil +from python.runfiles import runfiles + +def options(): + p = argparse.ArgumentParser(description="Update generated files related to Go in the repo") + p.add_argument("--force", "-f", action="store_true", help="Regenerate all files from scratch rather than updating them") + p.add_argument("executables", nargs=3, help="Internally provided executables") + return p.parse_args() + +opts = options() + +try: + workspace_dir = pathlib.Path(os.environ.pop('BUILD_WORKSPACE_DIRECTORY')) +except KeyError: + print("this should be run with bazel run", file=sys.stderr) + sys.exit(1) + +go_extractor_dir = workspace_dir / "go" / "extractor" + +if not go_extractor_dir.exists(): + # internal repo? + workspace_dir /= "ql" + go_extractor_dir = workspace_dir / "go" / "extractor" + +go_dbscheme = workspace_dir / "go" / "ql" / "lib" / "go.dbscheme" +r = runfiles.Create() +go, gazelle, go_gen_dbscheme = map(r.Rlocation, opts.executables) + + +if opts.force: + print("clearing vendor directory") + shutil.rmtree(go_extractor_dir / "vendor") + +existing_build_files = set(go_extractor_dir.glob("*/**/BUILD.bazel")) + +print("updating vendor directory") +subprocess.check_call([go, "-C", go_extractor_dir, "work", "vendor"]) + +if opts.force: + print("clearing generated BUILD files") + for build_file in existing_build_files: + build_file.unlink() + +print("running gazelle", gazelle, go_extractor_dir) +subprocess.check_call([gazelle, "go/extractor"], cwd=workspace_dir) + +# we want to stamp all newly generated `BUILD.bazel` files with a header +build_files_to_update = set(go_extractor_dir.glob("*/**/BUILD.bazel")) +# if --force, all files are new +if not opts.force: + # otherwise, subtract the files that existed at the start + build_files_to_update -= existing_build_files + # but bring back the `vendor` ones, as the vendor update step always clears them + build_files_to_update.update(go_extractor_dir.glob("vendor/**/BUILD.bazel")) + +print("adding header to newly generated BUILD files") +for build_file in build_files_to_update: + contents = build_file.read_text() + build_file.write_text(f"# generated running `bazel run //go/gazelle`, do not edit\n\n{contents}") + +subprocess.check_call([go_gen_dbscheme, go_dbscheme]) diff --git a/go/rules.bzl b/go/rules.bzl new file mode 100644 index 00000000000..e26dd57bf44 --- /dev/null +++ b/go/rules.bzl @@ -0,0 +1,37 @@ +load("@rules_go//go:def.bzl", "go_binary", "go_cross_binary") + +def codeql_go_binary(*, name, visibility = None, **kwargs): + def internal(prefix = "internal"): + return "%s/%s" % (prefix, name) + + go_binary( + name = internal(), + visibility = ["//visibility:private"], + **kwargs + ) + macos_targets = ("darwin_arm64", "darwin_amd64") + for target in macos_targets: + go_cross_binary( + name = internal(target), + platform = "@rules_go//go/toolchain:%s" % target, + target = internal(), + target_compatible_with = ["@platforms//os:macos"], + visibility = ["//visibility:private"], + ) + native.genrule( + name = internal("universal"), + outs = [internal("universal_")], + srcs = [internal(t) for t in macos_targets], + target_compatible_with = ["@platforms//os:macos"], + executable = True, + visibility = ["//visibility:private"], + cmd = "lipo -create $(SRCS) -output $@", + ) + native.alias( + name = name, + actual = select({ + "@platforms//os:macos": internal("universal"), + "//conditions:default": internal(), + }), + visibility = visibility, + )