Merge branch 'main' into remote-persistent-safe

Author: MarcusSorealheis

.github/workflows/native-cargo.yaml

@@ -41,6 +41,15 @@ jobs:
         run: rustup update && rustup default ${{ matrix.toolchain }}
         shell: bash
 
+      - name: Install protoc
+        run: |
+          if [ "$RUNNER_OS" == "Linux" ]; then
+            sudo apt-get update && sudo apt-get install -y protobuf-compiler
+          elif [ "$RUNNER_OS" == "Windows" ]; then
+            choco install protoc
+          fi
+        shell: bash
+
       - name: Rust cache
         # https://github.com/Swatinem/rust-cache/releases/tag/v2.8.1
         uses: Swatinem/rust-cache@a84bfdc502f07db5a85dd9d7a30f91a931516cc5

Cargo.lock

@@ -141,6 +141,7 @@
                 then [pkgs.mold]
                 else [pkgs.llvmPackages_20.lld]
               )
+              ++ [p.protobuf] # Required for nativelink-crio-worker-pool proto compilation
               ++ pkgs.lib.optionals p.stdenv.targetPlatform.isDarwin [
                 p.darwin.apple_sdk.frameworks.Security
                 p.libiconv

flake.nix

@@ -16,6 +16,7 @@ rust_library(
         "src/schedulers.rs",
         "src/serde_utils.rs",
         "src/stores.rs",
+        "src/warm_worker_pools.rs",
     ],
     compile_data = [
         "README.md",

nativelink-config/BUILD.bazel

@@ -10,7 +10,7 @@ version = "0.7.6"
 nativelink-error = { path = "../nativelink-error" }
 
 byte-unit = { version = "5.1.6", default-features = false, features = ["byte"] }
-humantime = { version = "2.2.0", default-features = false }
+humantime = { version = "2.3.0", default-features = false }
 rand = { version = "0.9.0", default-features = false, features = [
   "thread_rng",
 ] }

nativelink-config/Cargo.toml

@@ -17,7 +17,4 @@ pub mod cas_server;
 pub mod schedulers;
 pub mod serde_utils;
 pub mod stores;
-
-// Warm worker pools configuration (optional feature)
-#[cfg(feature = "warm-worker-pools")]
 pub mod warm_worker_pools;

nativelink-config/src/lib.rs

@@ -21,7 +21,6 @@ use crate::serde_utils::{
     convert_numeric_with_shellexpand,
 };
 use crate::stores::{GrpcEndpoint, Retry, StoreRefName};
-
 // Import warm worker pool configuration
 #[cfg(feature = "warm-worker-pools")]
 use crate::warm_worker_pools::WarmWorkerPoolsConfig;

nativelink-config/src/schedulers.rs

@@ -639,7 +639,7 @@ pub struct OntapS3ExistenceCacheSpec {
     pub backend: Box<ExperimentalOntapS3Spec>,
 }
 
-#[derive(Serialize, Deserialize, Default, Debug, Clone, Copy, PartialEq)]
+#[derive(Serialize, Deserialize, Default, Debug, Clone, Copy, PartialEq, Eq)]
 #[serde(rename_all = "snake_case")]
 pub enum StoreDirection {
     /// The store operates normally and all get and put operations are

nativelink-config/src/stores.rs

@@ -132,7 +132,7 @@ impl Default for LifecycleConfig {
 }
 
 /// Isolation strategy for worker jobs.
-#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)]
+#[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq)]
 #[serde(rename_all = "snake_case")]
 pub enum IsolationStrategy {
     /// No isolation - workers execute multiple jobs with shared state (default, backward compatible).

nativelink-config/src/warm_worker_pools.rs

@@ -1,26 +1,98 @@
+load("@bazel_skylib//lib:selects.bzl", "selects")
 load("@rules_rust//rust:defs.bzl", "rust_library", "rust_test")
 
+# Platform configurations for protoc
+PLATFORM_OS_ARCH = [
+    ("linux", "aarch64"),
+    ("linux", "x86_64"),
+    ("macos", "aarch64"),
+    ("macos", "x86_64"),
+    ("windows", "aarch64"),
+    ("windows", "x86_64"),
+]
+
+[
+    selects.config_setting_group(
+        name = "{}_{}".format(
+            os.replace("macos", "osx"),
+            arch.replace("aarch64", "aarch_64"),
+        ),
+        match_all = [
+            "@platforms//cpu:{}".format(arch),
+            "@platforms//os:{}".format(os),
+        ],
+    )
+    for (os, arch) in PLATFORM_OS_ARCH
+]
+
+PLATFORM_NAMES = [
+    "{}_{}".format(
+        os.replace("macos", "osx"),
+        arch.replace("aarch64", "aarch_64"),
+    )
+    for (os, arch) in PLATFORM_OS_ARCH
+]
+
+# Generate Rust code from CRI proto files
+genrule(
+    name = "gen_cri_protos",
+    srcs = ["proto/cri/api.proto"],
+    outs = ["runtime.v1.pb.rs"],
+    cmd = select({
+        platform: '''
+        set -e
+        export PROTOC=$(execpath @@toolchains_protoc++protoc+toolchains_protoc_hub.{}//:bin/protoc)
+
+        $(execpath //nativelink-proto:gen_protos_tool) $(SRCS) -o $(RULEDIR)
+
+        for file in $(RULEDIR)/*.rs; do
+            mv -- "$$file" "$${{file%.rs}}.pb.rs"
+        done
+        '''.format(platform)
+        for platform in PLATFORM_NAMES
+    }),
+    tools = [
+        "//nativelink-proto:gen_protos_tool",
+    ] + select({
+        platform: ["@@toolchains_protoc++protoc+toolchains_protoc_hub.{}//:bin/protoc".format(platform)]
+        for platform in PLATFORM_NAMES
+    }),
+)
+
 rust_library(
     name = "nativelink-crio-worker-pool",
     srcs = [
         "src/cache.rs",
         "src/config.rs",
         "src/cri_client.rs",
+        "src/cri_client_grpc.rs",
+        "src/isolation.rs",
         "src/lib.rs",
         "src/lifecycle.rs",
         "src/pool_manager.rs",
         "src/warmup.rs",
         "src/worker.rs",
+        ":gen_cri_protos",
+    ],
+    compile_data = [
+        ":gen_cri_protos",
     ],
+    tags = ["no-clippy"],
     visibility = ["//visibility:public"],
     deps = [
+        "//nativelink-config",
         "//nativelink-error",
+        "//nativelink-metric",
         "//nativelink-util",
+        "@crates//:hyper-util",
+        "@crates//:prost",
         "@crates//:serde",
         "@crates//:serde_json",
         "@crates//:serde_with",
         "@crates//:tempfile",
         "@crates//:tokio",
+        "@crates//:tonic",
+        "@crates//:tower",
         "@crates//:tracing",
         "@crates//:uuid",
     ],
@@ -29,4 +101,5 @@ rust_library(
 rust_test(
     name = "unit_tests",
     crate = ":nativelink-crio-worker-pool",
+    tags = ["no-clippy"],
 )