Merge pull request #15845 from github/tausbn/python-extractor-fix-build

Python: Build external extractor

python/BUILD.bazel

@@ -1,4 +1,6 @@
-load("@rules_pkg//:mappings.bzl", "pkg_filegroup", "pkg_files")
+load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup", "pkg_files")
+load("@semmle_code//:dist.bzl", "dist", "pack_zip")
+load("//:defs.bzl", "codeql_platform")
 
 package(default_visibility = ["//visibility:public"])
 
@@ -28,3 +30,32 @@ pkg_filegroup(
         "//python/downgrades",
     ],
 )
+
+pkg_files(
+    name = "codeql-extractor-yml",
+    srcs = ["codeql-extractor.yml"],
+    strip_prefix = None,
+)
+
+dist(
+    name = "extractor-generic",
+    srcs = [
+        ":codeql-extractor-yml",
+        ":dbscheme-group",
+        "//python/downgrades",
+        "//python/extractor",
+        "//python/tools",
+    ],
+    prefix = "python",
+    visibility = ["//visibility:public"],
+)
+
+pack_zip(
+    name = "extractor-arch",
+    srcs = [
+        "//python/extractor/tsg-python",
+    ],
+    package_file_name = "extractor-" + codeql_platform + ".zip",
+    prefix = "python/tools/" + codeql_platform,
+    visibility = ["//visibility:public"],
+)

python/codeql-extractor.yml

@@ -0,0 +1,46 @@
+name: "python"
+display_name: "Python"
+version: 1.22.1
+column_kind: utf32
+build_modes:
+  - none
+github_api_languages:
+  - Python
+scc_languages:
+  - Python
+file_types:
+  - name: python
+    display_name: Python sources
+    extensions:
+      - .py
+legacy_qltest_extraction: true
+options:
+  logging:
+    title: Options pertaining to logging.
+    type: object
+    properties:
+      verbosity:
+        title: Python extractor logging verbosity level.
+        description: >
+          Controls the level of verbosity of the CodeQL Python extractor.
+
+          The supported levels are (in order of increasing verbosity):
+
+            - off
+            - errors
+            - warnings
+            - info or progress
+            - debug or progress+
+            - trace or progress++
+            - progress+++
+        type: string
+        pattern: "^(off|errors|warnings|(info|progress)|(debug|progress\\+)|(trace|progress\\+\\+)|progress\\+\\+\\+)$"
+  python_executable_name:
+    title: Controls the name of the Python executable used by the Python extractor.
+    description: >
+      The Python extractor uses platform-dependent heuristics to determine the name of the Python executable to use.
+      Specifying a value for this option overrides the name of the Python executable used by the extractor.
+      Accepted values are py, python and python3.
+      Use this setting with caution, the Python extractor requires Python 3 to run.
+    type: string
+    pattern: "^(py|python|python3)$"

python/extractor/BUILD.bazel

@@ -1,4 +1,4 @@
-load("//:dist.bzl", "pack_zip")
+load("@semmle_code//:dist.bzl", "pack_zip")
 
 py_binary(
     name = "make-zips-py",
@@ -33,7 +33,7 @@ genrule(
 )
 
 pack_zip(
-    name = "extractor-python",
+    name = "extractor",
     srcs = [
         "LICENSE-PSF.md",  # because we distribute imp.py
         "convert_setup.py",

python/extractor/cli-integration-test/writing-diagnostics/test_diagnostics_output.py

@@ -1,6 +1,6 @@
 import os
 import sys
-sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "..", "integration-tests"))
+sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "..", "integration-tests"))
 import diagnostics_test_utils
 
 test_db = "db"

python/extractor/qlpack.yml

@@ -1,6 +1,6 @@
 name: extractor-python
 dependencies:
-  codeql/python-all: "*"
-  codeql/python-queries: "*"
+  codeql/python-all: ${workspace}
+  codeql/python-queries: ${workspace}
 extractor: python
 warnOnImplicitThis: true

python/extractor/tests/test_patterns.py

@@ -12,7 +12,7 @@ class ExtractorPatternsTest(test_utils.ExtractorTest):
 
     def test(self):
         repo_dir = subprocess.Popen(["git", "rev-parse", "--show-toplevel"], stdout=subprocess.PIPE).communicate()[0].rstrip().decode("utf-8")
-        test_file_path = os.path.abspath(os.path.join(repo_dir, "unit-tests", "files", "pattern-matching", "patterns.json"))
+        test_file_path = os.path.abspath(os.path.join(repo_dir, "..", "unit-tests", "files", "pattern-matching", "patterns.json"))
         with open(test_file_path) as test_file:
             test_patterns = json.load(test_file)
         for test_pattern in test_patterns:

python/extractor/tsg-python/BUILD.bazel

@@ -1,5 +1,5 @@
-load("@tsg_python_crate_index//:defs.bzl", "aliases", "all_crate_deps")
-load("//:common.bzl", "codeql_rust_binary")
+load("@py_deps//:defs.bzl", "aliases", "all_crate_deps")
+load("@semmle_code//:common.bzl", "codeql_rust_binary")
 
 codeql_rust_binary(
     name = "tsg-python",
@@ -12,5 +12,5 @@ codeql_rust_binary(
     visibility = ["//visibility:public"],
     deps = all_crate_deps(
         normal = True,
-    ) + ["//extractor-python/tsg-python/tree-sitter-python"],
+    ) + ["//python/extractor/tsg-python/tsp"],
 )

python/extractor/tsg-python/Cargo.Bazel.lock

@@ -1,5 +1,5 @@
 {
-  "checksum": "54f1095f5a2e74da736682bc8d355b3dbce47558983feba04faba84cf3abfaca",
+  "checksum": "35a1ce4b6c4f997c496c11d3a8fcfaadc5833dfd41bebb022941687d73dde159",
   "crates": {
     "ahash 0.4.7": {
       "name": "ahash",
@@ -7,7 +7,7 @@
       "package_url": "https://github.com/tkaitchuck/ahash",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/ahash/0.4.7/download",
+          "url": "https://static.crates.io/crates/ahash/0.4.7/download",
           "sha256": "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e"
         }
       },
@@ -43,7 +43,7 @@
       "package_url": "https://github.com/BurntSushi/aho-corasick",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/aho-corasick/0.7.18/download",
+          "url": "https://static.crates.io/crates/aho-corasick/0.7.18/download",
           "sha256": "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f"
         }
       },
@@ -95,7 +95,7 @@
       "package_url": "https://github.com/ogham/rust-ansi-term",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/ansi_term/0.11.0/download",
+          "url": "https://static.crates.io/crates/ansi_term/0.11.0/download",
           "sha256": "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
         }
       },
@@ -141,7 +141,7 @@
       "package_url": "https://github.com/dtolnay/anyhow",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/anyhow/1.0.44/download",
+          "url": "https://static.crates.io/crates/anyhow/1.0.44/download",
           "sha256": "61604a8f862e1d5c3229fdd78f8b02c68dcf73a4c4b05fd636d12240aaa242c1"
         }
       },
@@ -207,7 +207,7 @@
       "package_url": "https://github.com/softprops/atty",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/atty/0.2.14/download",
+          "url": "https://static.crates.io/crates/atty/0.2.14/download",
           "sha256": "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
         }
       },
@@ -265,7 +265,7 @@
       "package_url": "https://github.com/bitflags/bitflags",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/bitflags/1.3.2/download",
+          "url": "https://static.crates.io/crates/bitflags/1.3.2/download",
           "sha256": "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
         }
       },
@@ -307,7 +307,7 @@
       "package_url": "https://github.com/alexcrichton/cc-rs",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/cc/1.0.70/download",
+          "url": "https://static.crates.io/crates/cc/1.0.70/download",
           "sha256": "d26a6ce4b6a484fa3edb70f7efa6fc430fd2b87285fe8b84304fd0936faa0dc0"
         }
       },
@@ -343,7 +343,7 @@
       "package_url": "https://github.com/alexcrichton/cfg-if",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/cfg-if/1.0.0/download",
+          "url": "https://static.crates.io/crates/cfg-if/1.0.0/download",
           "sha256": "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
         }
       },
@@ -379,7 +379,7 @@
       "package_url": "https://github.com/clap-rs/clap",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/clap/2.33.3/download",
+          "url": "https://static.crates.io/crates/clap/2.33.3/download",
           "sha256": "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002"
         }
       },
@@ -462,7 +462,7 @@
       "package_url": "https://github.com/rust-lang/hashbrown",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/hashbrown/0.9.1/download",
+          "url": "https://static.crates.io/crates/hashbrown/0.9.1/download",
           "sha256": "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
         }
       },
@@ -514,7 +514,7 @@
       "package_url": "https://github.com/hermitcore/libhermit-rs",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/hermit-abi/0.1.19/download",
+          "url": "https://static.crates.io/crates/hermit-abi/0.1.19/download",
           "sha256": "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
         }
       },
@@ -559,7 +559,7 @@
       "package_url": "https://github.com/dtolnay/itoa",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/itoa/1.0.1/download",
+          "url": "https://static.crates.io/crates/itoa/1.0.1/download",
           "sha256": "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35"
         }
       },
@@ -595,7 +595,7 @@
       "package_url": "https://github.com/rust-lang/libc",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/libc/0.2.101/download",
+          "url": "https://static.crates.io/crates/libc/0.2.101/download",
           "sha256": "3cb00336871be5ed2c8ed44b60ae9959dc5b9f08539422ed43f09e34ecaeba21"
         }
       },
@@ -654,7 +654,7 @@
       "package_url": "https://github.com/rust-lang/log",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/log/0.4.14/download",
+          "url": "https://static.crates.io/crates/log/0.4.14/download",
           "sha256": "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
         }
       },
@@ -717,7 +717,7 @@
       "package_url": "https://github.com/BurntSushi/memchr",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/memchr/2.4.1/download",
+          "url": "https://static.crates.io/crates/memchr/2.4.1/download",
           "sha256": "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
         }
       },
@@ -783,7 +783,7 @@
       "package_url": "https://github.com/alexcrichton/proc-macro2",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/proc-macro2/1.0.29/download",
+          "url": "https://static.crates.io/crates/proc-macro2/1.0.29/download",
           "sha256": "b9f5105d4fdaab20335ca9565e106a5d9b82b6219b5ba735731124ac6711d23d"
         }
       },
@@ -853,7 +853,7 @@
       "package_url": "https://github.com/dtolnay/quote",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/quote/1.0.9/download",
+          "url": "https://static.crates.io/crates/quote/1.0.9/download",
           "sha256": "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
         }
       },
@@ -905,7 +905,7 @@
       "package_url": "https://github.com/rust-lang/regex",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/regex/1.5.5/download",
+          "url": "https://static.crates.io/crates/regex/1.5.5/download",
           "sha256": "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286"
         }
       },
@@ -980,7 +980,7 @@
       "package_url": "https://github.com/rust-lang/regex",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/regex-syntax/0.6.25/download",
+          "url": "https://static.crates.io/crates/regex-syntax/0.6.25/download",
           "sha256": "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
         }
       },
@@ -1030,7 +1030,7 @@
       "package_url": "https://github.com/dtolnay/ryu",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/ryu/1.0.9/download",
+          "url": "https://static.crates.io/crates/ryu/1.0.9/download",
           "sha256": "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f"
         }
       },
@@ -1066,7 +1066,7 @@
       "package_url": "https://github.com/serde-rs/serde",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/serde/1.0.136/download",
+          "url": "https://static.crates.io/crates/serde/1.0.136/download",
           "sha256": "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789"
         }
       },
@@ -1132,7 +1132,7 @@
       "package_url": "https://github.com/serde-rs/json",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/serde_json/1.0.79/download",
+          "url": "https://static.crates.io/crates/serde_json/1.0.79/download",
           "sha256": "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95"
         }
       },
@@ -1210,7 +1210,7 @@
       "package_url": "https://github.com/servo/rust-smallvec",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/smallvec/1.6.1/download",
+          "url": "https://static.crates.io/crates/smallvec/1.6.1/download",
           "sha256": "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e"
         }
       },
@@ -1252,7 +1252,7 @@
       "package_url": "https://github.com/robbepop/string-interner",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/string-interner/0.12.2/download",
+          "url": "https://static.crates.io/crates/string-interner/0.12.2/download",
           "sha256": "383196d1876517ee6f9f0864d1fc1070331b803335d3c6daaa04bbcccd823c08"
         }
       },
@@ -1309,7 +1309,7 @@
       "package_url": "https://github.com/dguo/strsim-rs",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/strsim/0.8.0/download",
+          "url": "https://static.crates.io/crates/strsim/0.8.0/download",
           "sha256": "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
         }
       },
@@ -1344,7 +1344,7 @@
       "package_url": "https://github.com/dtolnay/syn",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/syn/1.0.76/download",
+          "url": "https://static.crates.io/crates/syn/1.0.76/download",
           "sha256": "c6f107db402c2c2055242dbf4d2af0e69197202e9faacbef9571bbe47f5a1b84"
         }
       },
@@ -1427,7 +1427,7 @@
       "package_url": "https://github.com/mgeisler/textwrap",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/textwrap/0.11.0/download",
+          "url": "https://static.crates.io/crates/textwrap/0.11.0/download",
           "sha256": "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
         }
       },
@@ -1471,7 +1471,7 @@
       "package_url": "https://github.com/dtolnay/thiserror",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/thiserror/1.0.29/download",
+          "url": "https://static.crates.io/crates/thiserror/1.0.29/download",
           "sha256": "602eca064b2d83369e2b2f34b09c70b605402801927c65c11071ac911d299b88"
         }
       },
@@ -1516,7 +1516,7 @@
       "package_url": "https://github.com/dtolnay/thiserror",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/thiserror-impl/1.0.29/download",
+          "url": "https://static.crates.io/crates/thiserror-impl/1.0.29/download",
           "sha256": "bad553cc2c78e8de258400763a647e80e6d1b31ee237275d756f6836d204494c"
         }
       },
@@ -1569,7 +1569,7 @@
       "package_url": "https://github.com/tree-sitter/tree-sitter",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/tree-sitter/0.20.4/download",
+          "url": "https://static.crates.io/crates/tree-sitter/0.20.4/download",
           "sha256": "4e34327f8eac545e3f037382471b2b19367725a242bba7bc45edb9efb49fe39a"
         }
       },
@@ -1640,7 +1640,7 @@
       "package_url": "https://github.com/tree-sitter/tree-sitter-graph/",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/tree-sitter-graph/0.7.0/download",
+          "url": "https://static.crates.io/crates/tree-sitter-graph/0.7.0/download",
           "sha256": "639d21e886f581d293de5f5081f09af003c54607ff3fa85efa159b243ba1f97a"
         }
       },
@@ -1707,72 +1707,6 @@
       ],
       "license_file": null
     },
-    "tree-sitter-python 0.19.0": {
-      "name": "tree-sitter-python",
-      "version": "0.19.0",
-      "package_url": "https://github.com/tree-sitter/tree-sitter-python",
-      "repository": null,
-      "targets": [
-        {
-          "Library": {
-            "crate_name": "tree_sitter_python",
-            "crate_root": "bindings/rust/lib.rs",
-            "srcs": [
-              "**/*.rs"
-            ]
-          }
-        },
-        {
-          "BuildScript": {
-            "crate_name": "build_script_build",
-            "crate_root": "bindings/rust/build.rs",
-            "srcs": [
-              "**/*.rs"
-            ]
-          }
-        }
-      ],
-      "library_target_name": "tree_sitter_python",
-      "common_attrs": {
-        "compile_data_glob": [
-          "**"
-        ],
-        "deps": {
-          "common": [
-            {
-              "id": "tree-sitter 0.20.4",
-              "target": "tree_sitter"
-            },
-            {
-              "id": "tree-sitter-python 0.19.0",
-              "target": "build_script_build"
-            }
-          ],
-          "selects": {}
-        },
-        "edition": "2018",
-        "version": "0.19.0"
-      },
-      "build_script_attrs": {
-        "data_glob": [
-          "**"
-        ],
-        "deps": {
-          "common": [
-            {
-              "id": "cc 1.0.70",
-              "target": "cc"
-            }
-          ],
-          "selects": {}
-        }
-      },
-      "license": "MIT",
-      "license_ids": [
-        "MIT"
-      ],
-      "license_file": null
-    },
     "tsg-python 0.1.0": {
       "name": "tsg-python",
       "version": "0.1.0",
@@ -1828,13 +1762,79 @@
       "license_ids": [],
       "license_file": null
     },
+    "tsp 0.19.0": {
+      "name": "tsp",
+      "version": "0.19.0",
+      "package_url": "https://github.com/tree-sitter/tree-sitter-python",
+      "repository": null,
+      "targets": [
+        {
+          "Library": {
+            "crate_name": "tsp",
+            "crate_root": "bindings/rust/lib.rs",
+            "srcs": [
+              "**/*.rs"
+            ]
+          }
+        },
+        {
+          "BuildScript": {
+            "crate_name": "build_script_build",
+            "crate_root": "bindings/rust/build.rs",
+            "srcs": [
+              "**/*.rs"
+            ]
+          }
+        }
+      ],
+      "library_target_name": "tsp",
+      "common_attrs": {
+        "compile_data_glob": [
+          "**"
+        ],
+        "deps": {
+          "common": [
+            {
+              "id": "tree-sitter 0.20.4",
+              "target": "tree_sitter"
+            },
+            {
+              "id": "tsp 0.19.0",
+              "target": "build_script_build"
+            }
+          ],
+          "selects": {}
+        },
+        "edition": "2018",
+        "version": "0.19.0"
+      },
+      "build_script_attrs": {
+        "data_glob": [
+          "**"
+        ],
+        "deps": {
+          "common": [
+            {
+              "id": "cc 1.0.70",
+              "target": "cc"
+            }
+          ],
+          "selects": {}
+        }
+      },
+      "license": "MIT",
+      "license_ids": [
+        "MIT"
+      ],
+      "license_file": null
+    },
     "unicode-width 0.1.8": {
       "name": "unicode-width",
       "version": "0.1.8",
       "package_url": "https://github.com/unicode-rs/unicode-width",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/unicode-width/0.1.8/download",
+          "url": "https://static.crates.io/crates/unicode-width/0.1.8/download",
           "sha256": "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3"
         }
       },
@@ -1876,7 +1876,7 @@
       "package_url": "https://github.com/unicode-rs/unicode-xid",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/unicode-xid/0.2.2/download",
+          "url": "https://static.crates.io/crates/unicode-xid/0.2.2/download",
           "sha256": "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
         }
       },
@@ -1918,7 +1918,7 @@
       "package_url": "https://github.com/contain-rs/vec-map",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/vec_map/0.8.2/download",
+          "url": "https://static.crates.io/crates/vec_map/0.8.2/download",
           "sha256": "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
         }
       },
@@ -1954,7 +1954,7 @@
       "package_url": "https://github.com/retep998/winapi-rs",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/winapi/0.3.9/download",
+          "url": "https://static.crates.io/crates/winapi/0.3.9/download",
           "sha256": "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
         }
       },
@@ -2037,7 +2037,7 @@
       "package_url": "https://github.com/retep998/winapi-rs",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/winapi-i686-pc-windows-gnu/0.4.0/download",
+          "url": "https://static.crates.io/crates/winapi-i686-pc-windows-gnu/0.4.0/download",
           "sha256": "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
         }
       },
@@ -2096,7 +2096,7 @@
       "package_url": "https://github.com/retep998/winapi-rs",
       "repository": {
         "Http": {
-          "url": "https://crates.io/api/v1/crates/winapi-x86_64-pc-windows-gnu/0.4.0/download",
+          "url": "https://static.crates.io/crates/winapi-x86_64-pc-windows-gnu/0.4.0/download",
           "sha256": "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
         }
       },
@@ -2152,8 +2152,8 @@
   },
   "binary_crates": [],
   "workspace_members": {
-    "tree-sitter-python 0.19.0": "extractor-python/tsg-python/tree-sitter-python",
-    "tsg-python 0.1.0": "extractor-python/tsg-python"
+    "tsg-python 0.1.0": "python/extractor/tsg-python",
+    "tsp 0.19.0": "python/extractor/tsg-python/tsp"
   },
   "conditions": {
     "aarch64-apple-darwin": [

python/extractor/tsg-python/Cargo.lock

@@ -267,14 +267,6 @@ dependencies = [
  "tree-sitter",
 ]
 
-[[package]]
-name = "tree-sitter-python"
-version = "0.19.0"
-dependencies = [
- "cc",
- "tree-sitter",
-]
-
 [[package]]
 name = "tsg-python"
 version = "0.1.0"
@@ -287,7 +279,15 @@ dependencies = [
  "thiserror",
  "tree-sitter",
  "tree-sitter-graph",
- "tree-sitter-python",
+ "tsp",
+]
+
+[[package]]
+name = "tsp"
+version = "0.19.0"
+dependencies = [
+ "cc",
+ "tree-sitter",
 ]
 
 [[package]]

python/extractor/tsg-python/Cargo.toml

@@ -8,7 +8,7 @@ edition = "2018"
 
 # When changing/updating these, the `Cargo.Bazel.lock` file has to be regenerated.
 # Check out the documentation at https://bazelbuild.github.io/rules_rust/crate_universe.html#repinning--updating-dependencies
-# for how to do so. The bazel repository for the tsg-python project is called `tsg_python_crate_index`,
+# for how to do so. The bazel repository for the tsg-python project is called `py_deps`,
 # and instead of calling `bazel sync`, `./build --bazel sync` should be used instead, to always use the correct bazel version.
 [dependencies]
 anyhow = "1.0"
@@ -17,7 +17,7 @@ smallvec = { version="1.6", features=["union"] }
 thiserror = "1.0"
 tree-sitter = "0.20.4"
 tree-sitter-graph = "0.7.0"
-tree-sitter-python = {path = "tree-sitter-python"}
+tsp = {path = "tsp"}
 clap = "2.32"
 
 [dependencies.string-interner]

python/extractor/tsg-python/src/main.rs

@@ -488,7 +488,7 @@ fn main() -> Result<()> {
         "bundled `python.tsg`".to_owned()
     };
     let source_path = Path::new(matches.value_of("source").unwrap());
-    let language = tree_sitter_python::language();
+    let language = tsp::language();
     let mut parser = Parser::new();
     parser.set_language(language)?;
     // Statically include `python.tsg`:

python/extractor/tsg-python/tsp/BUILD.bazel

@@ -1,13 +1,13 @@
 load("@rules_rust//cargo:defs.bzl", "cargo_build_script")
 load("@rules_rust//rust:defs.bzl", "rust_library")
-load("@tsg_python_crate_index//:defs.bzl", "aliases", "all_crate_deps")
+load("@py_deps//:defs.bzl", "aliases", "all_crate_deps")
 
 package(default_visibility = ["//visibility:public"])
 
 # This will run the build script from the root of the workspace, and
 # collect the outputs.
 cargo_build_script(
-    name = "tsg-build-script",
+    name = "tsg-build",
     srcs = ["bindings/rust/build.rs"],
     data = glob([
         "src/**",
@@ -18,7 +18,7 @@ cargo_build_script(
 )
 
 rust_library(
-    name = "tree-sitter-python",
+    name = "tsp",
     srcs = [
         "bindings/rust/lib.rs",
     ],
@@ -32,7 +32,7 @@ rust_library(
     proc_macro_deps = all_crate_deps(
         proc_macro = True,
     ),
-    deps = [":tsg-build-script"] + all_crate_deps(
+    deps = [":tsg-build"] + all_crate_deps(
         normal = True,
     ),
 )

python/extractor/tsg-python/tsp/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "tree-sitter-python"
+name = "tsp"
 description = "Python grammar for the tree-sitter parsing library"
 version = "0.19.0"
 authors = [

python/tools/BUILD.bazel

@@ -0,0 +1,11 @@
+load("@semmle_code//:dist.bzl", "pack_zip")
+
+pack_zip(
+    name = "tools",
+    srcs = glob(["**/*"]),
+    excludes = [
+        "BUILD.bazel",
+    ] + glob(["recorded-call-graph-metrics/**"]),
+    prefix = "tools",
+    visibility = ["//visibility:public"],
+)

python/tools/autobuild.cmd

@@ -0,0 +1,8 @@
+@echo off
+
+rem Legacy environment variables for the autobuild infrastructure.
+set LGTM_SRC=%CD%
+set LGTM_WORKSPACE=%CODEQL_EXTRACTOR_PYTHON_SCRATCH_DIR%
+
+type NUL && python "%CODEQL_EXTRACTOR_PYTHON_ROOT%\tools\index.py"
+exit /b %ERRORLEVEL%

python/tools/autobuild.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+set -eu
+
+# Legacy environment variables for the autobuild infrastructure.
+LGTM_SRC="$(pwd)"
+LGTM_WORKSPACE="$CODEQL_EXTRACTOR_PYTHON_SCRATCH_DIR"
+export LGTM_SRC
+export LGTM_WORKSPACE
+
+if which python3 >/dev/null; then
+    exec python3 "$CODEQL_EXTRACTOR_PYTHON_ROOT/tools/index.py"
+elif which python >/dev/null; then
+    exec python "$CODEQL_EXTRACTOR_PYTHON_ROOT/tools/index.py"
+else
+    echo "ERROR: Could not find a valid Python distribution. It should be available when running 'which python' or 'which python3' in your shell. Python 2 is no longer supported."
+    exit 1
+fi

python/tools/lgtm-scripts/index.cmd

@@ -0,0 +1,3 @@
+@echo off
+
+py "%CODEQL_EXTRACTOR_PYTHON_ROOT%\tools\index.py"

python/tools/lgtm-scripts/index.sh

@@ -0,0 +1,5 @@
+#! /bin/bash
+
+set -eu
+
+python "${CODEQL_EXTRACTOR_PYTHON_ROOT}/tools/index.py"

python/tools/lgtm-scripts/python_setup.cmd

@@ -0,0 +1,3 @@
+@echo off
+
+py "%CODEQL_EXTRACTOR_PYTHON_ROOT%\tools\setup.py" || EXIT /B 0

python/tools/lgtm-scripts/python_setup.sh

@@ -0,0 +1,5 @@
+#! /bin/bash
+
+set -eu
+
+python "${CODEQL_EXTRACTOR_PYTHON_ROOT}/tools/setup.py" || true

python/tools/pre-finalize.cmd

@@ -0,0 +1,11 @@
+@echo off
+
+type NUL && "%CODEQL_DIST%\codeql" database index-files ^
+    --include-extension=.yaml ^
+    --include-extension=.yml ^
+    --size-limit=5m ^
+    --language yaml ^
+    -- ^
+    "%CODEQL_EXTRACTOR_PYTHON_WIP_DATABASE%"
+
+exit /b %ERRORLEVEL%

python/tools/pre-finalize.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -eu
+
+"$CODEQL_DIST/codeql" database index-files \
+    --include-extension=.yaml \
+    --include-extension=.yml \
+    --size-limit=5m \
+    --language yaml \
+    -- \
+    "$CODEQL_EXTRACTOR_PYTHON_WIP_DATABASE"