Rust: Vendor 3rdparty dependencies.

We've been observing some performance issues using crate_universe on CI.
Therefore, we're moving to vendor the auto-generated BUILD files
in our repository. This should provide a nice speed boost, while
getting rid of the complexity of the "rust cache" job we've been using
when we had a lot of git dependencies.

This PR includes a vendor script, and I'll put up a CI job internally
that runs that vendor script on Cargo.toml and Cargo.lock changes, to check
that the vendored files are in sync.

Author: criemen

.bazelrc

@@ -23,6 +23,5 @@ common --registry=file:///%workspace%/misc/bazel/registry
 common --registry=https://bcr.bazel.build
 
 common --@rules_dotnet//dotnet/settings:strict_deps=false
-common --experimental_isolated_extension_usages
 
 try-import %workspace%/local.bazelrc

.bazelrc.internal

@@ -8,4 +8,3 @@ common --registry=https://bcr.bazel.build
 # its implementation packages without providing any code itself.
 # We either can depend on internal implementation details, or turn of strict deps.
 common --@rules_dotnet//dotnet/settings:strict_deps=false
-common --experimental_isolated_extension_usages

BUILD.bazel

@@ -1 +1,5 @@
-exports_files(["LICENSE"])
+exports_files([
+    "LICENSE",
+    "Cargo.lock",
+    "Cargo.toml",
+])

MODULE.bazel

@@ -31,41 +31,44 @@ bazel_dep(name = "rules_rust", version = "0.52.2")
 
 bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True)
 
-# crate_py but shortened due to Windows file path considerations
-cp = use_extension(
-    "@rules_rust//crate_universe:extension.bzl",
-    "crate",
-    isolate = True,
-)
-cp.from_cargo(
-    name = "py_deps",
-    cargo_lockfile = "//python/extractor/tsg-python:Cargo.lock",
-    manifests = [
-        "//python/extractor/tsg-python:Cargo.toml",
-        "//python/extractor/tsg-python/tsp:Cargo.toml",
+# Keep edition and version approximately in sync with internal repo.
+# the versions there are canonical, the versions here are used for CI in github/codeql, as well as for the vendoring of dependencies.
+RUST_EDITION = "2021"
+
+RUST_VERSION = "1.81.0"
+
+rust = use_extension("@rules_rust//rust:extensions.bzl", "rust")
+rust.toolchain(
+    edition = RUST_EDITION,
+    # We need those extra target triples so that we can build universal binaries on macos
+    extra_target_triples = [
+        "x86_64-apple-darwin",
+        "aarch64-apple-darwin",
     ],
+    versions = [RUST_VERSION],
 )
-use_repo(cp, "py_deps")
+use_repo(rust, "rust_toolchains")
 
-# deps for ruby+rust, but shortened due to windows file paths
-r = use_extension(
-    "@rules_rust//crate_universe:extension.bzl",
-    "crate",
-    isolate = True,
-)
-r.from_cargo(
-    name = "r",
-    cargo_lockfile = "//:Cargo.lock",
-    manifests = [
-        "//:Cargo.toml",
-        "//ruby/extractor:Cargo.toml",
-        "//rust/extractor:Cargo.toml",
-        "//rust/extractor/macros:Cargo.toml",
-        "//rust/ast-generator:Cargo.toml",
-        "//shared/tree-sitter-extractor:Cargo.toml",
-    ],
+register_toolchains("@rust_toolchains//:all")
+
+rust_host_tools = use_extension("@rules_rust//rust:extensions.bzl", "rust_host_tools")
+
+# Don't download a second toolchain as host toolchain, make sure this is the same version as above
+# The host toolchain is used for vendoring dependencies.
+rust_host_tools.host_tools(
+    edition = RUST_EDITION,
+    version = RUST_VERSION,
 )
-use_repo(r, tree_sitter_extractors_deps = "r")
+
+# deps for python extractor
+# keep in sync by running `misc/bazel/3rdparty/update_cargo_deps.sh`
+py_deps = use_extension("//misc/bazel/3rdparty:py_deps_extension.bzl", "p")
+use_repo(py_deps, "vendor__anyhow-1.0.44", "vendor__cc-1.0.70", "vendor__clap-2.33.3", "vendor__regex-1.5.5", "vendor__smallvec-1.6.1", "vendor__string-interner-0.12.2", "vendor__thiserror-1.0.29", "vendor__tree-sitter-0.20.4", "vendor__tree-sitter-graph-0.7.0")
+
+# deps for ruby+rust
+# keep in sync by running `misc/bazel/3rdparty/update_cargo_deps.sh`
+tree_sitter_extractors_deps = use_extension("//misc/bazel/3rdparty:tree_sitter_extractors_extension.bzl", "r")
+use_repo(tree_sitter_extractors_deps, "vendor__anyhow-1.0.93", "vendor__argfile-0.2.1", "vendor__chrono-0.4.38", "vendor__clap-4.5.20", "vendor__encoding-0.2.33", "vendor__figment-0.10.19", "vendor__flate2-1.0.34", "vendor__glob-0.3.1", "vendor__globset-0.4.15", "vendor__itertools-0.10.5", "vendor__itertools-0.13.0", "vendor__lazy_static-1.5.0", "vendor__log-0.4.22", "vendor__num-traits-0.2.19", "vendor__num_cpus-1.16.0", "vendor__proc-macro2-1.0.89", "vendor__quote-1.0.37", "vendor__ra_ap_base_db-0.0.232", "vendor__ra_ap_hir-0.0.232", "vendor__ra_ap_hir_def-0.0.232", "vendor__ra_ap_hir_expand-0.0.232", "vendor__ra_ap_ide_db-0.0.232", "vendor__ra_ap_load-cargo-0.0.232", "vendor__ra_ap_parser-0.0.232", "vendor__ra_ap_paths-0.0.232", "vendor__ra_ap_project_model-0.0.232", "vendor__ra_ap_span-0.0.232", "vendor__ra_ap_syntax-0.0.232", "vendor__ra_ap_vfs-0.0.232", "vendor__rand-0.8.5", "vendor__rayon-1.10.0", "vendor__regex-1.11.1", "vendor__serde-1.0.214", "vendor__serde_json-1.0.132", "vendor__serde_with-3.11.0", "vendor__stderrlog-0.6.0", "vendor__syn-2.0.87", "vendor__tracing-0.1.40", "vendor__tracing-subscriber-0.3.18", "vendor__tree-sitter-0.24.4", "vendor__tree-sitter-embedded-template-0.23.2", "vendor__tree-sitter-json-0.24.8", "vendor__tree-sitter-ql-0.23.1", "vendor__tree-sitter-ruby-0.23.1", "vendor__triomphe-0.1.14", "vendor__ungrammar-1.16.1")
 
 dotnet = use_extension("@rules_dotnet//dotnet:extensions.bzl", "dotnet")
 dotnet.toolchain(dotnet_version = "8.0.101")

misc/bazel/3rdparty/.gitattributes

@@ -0,0 +1,2 @@
+/tree_sitter_extractors_deps/** linguist-generated=true
+/py_deps/** linguist-generated=true

misc/bazel/3rdparty/BUILD.bazel

@@ -0,0 +1,34 @@
+load("@rules_rust//crate_universe:defs.bzl", "crates_vendor")
+
+# The targets in this file re-do the vendoring of the rust cargo dependencies. Use the update_cargo_deps.sh script to run the targets, and do all the necessary work around vendoring.
+crates_vendor(
+    name = "vendor_tree_sitter_extractors",
+    cargo_lockfile = "//:Cargo.lock",
+    generate_build_scripts = True,
+    manifests = [
+        "//:Cargo.toml",
+        "//ruby/extractor:Cargo.toml",
+        "//rust/extractor:Cargo.toml",
+        "//rust/extractor/macros:Cargo.toml",
+        "//rust/ast-generator:Cargo.toml",
+        "//shared/tree-sitter-extractor:Cargo.toml",
+    ],
+    mode = "remote",
+    repository_name = "vendor",
+    tags = ["manual"],
+    vendor_path = "tree_sitter_extractors_deps",
+)
+
+crates_vendor(
+    name = "vendor_py_deps",
+    cargo_lockfile = "//python/extractor/tsg-python:Cargo.lock",
+    generate_build_scripts = True,
+    manifests = [
+        "//python/extractor/tsg-python:Cargo.toml",
+        "//python/extractor/tsg-python/tsp:Cargo.toml",
+    ],
+    mode = "remote",
+    repository_name = "vendor",
+    tags = ["manual"],
+    vendor_path = "py_deps",
+)