[Rust Frontend][gRPC] Add RL lifecycle control (#51316)

Signed-off-by: Connor Carpenter <connorc@nvidia.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Connor Carpenter
2026-08-14 11:32:39 -07:00
committed by GitHub
co-authored by Nick Hill
parent 3e3ceb1961
commit f473870ecf
17 changed files with 614 additions and 16 deletions
+73
View File
@@ -1,12 +1,14 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::collections::BTreeMap;
use std::sync::Arc;
use std::time::Duration;
use futures::future::{join_all, try_join_all};
use itertools::Itertools;
use serde::Serialize;
use serde_json::Value as JsonValue;
use tokio::sync::mpsc;
use tokio_util::task::AbortOnDropHandle;
use tracing::{debug, info, trace};
@@ -674,6 +676,77 @@ impl EngineCoreClient {
.collect())
}
/// Initialize the configured RL weight-transfer backend.
pub async fn init_weight_transfer_engine(&self, init_info: JsonValue) -> Result<()> {
self.collective_rpc(
"init_weight_transfer_engine",
None,
Vec::<JsonValue>::new(),
BTreeMap::from([("init_info".to_string(), init_info)]),
)
.await?;
Ok(())
}
/// Start a weight update for the base model.
pub async fn start_weight_update(&self) -> Result<()> {
self.collective_rpc(
"start_weight_update",
None,
Vec::<JsonValue>::new(),
BTreeMap::<String, JsonValue>::new(),
)
.await?;
Ok(())
}
/// Start a weight update for the speculative draft model.
pub async fn start_draft_weight_update(&self) -> Result<()> {
self.collective_rpc(
"start_draft_weight_update",
None,
Vec::<JsonValue>::new(),
BTreeMap::<String, JsonValue>::new(),
)
.await?;
Ok(())
}
/// Apply one backend-specific weight metadata chunk.
pub async fn update_weights(&self, update_info: JsonValue) -> Result<()> {
self.collective_rpc(
"update_weights",
None,
Vec::<JsonValue>::new(),
BTreeMap::from([("update_info".to_string(), update_info)]),
)
.await?;
Ok(())
}
/// Finish the current weight update.
pub async fn finish_weight_update(&self) -> Result<()> {
self.collective_rpc(
"finish_weight_update",
None,
Vec::<JsonValue>::new(),
BTreeMap::<String, JsonValue>::new(),
)
.await?;
Ok(())
}
/// Set the committed weight version on every connected engine.
pub async fn set_weight_version(&self, weight_version: &str) -> Result<()> {
self.call_utility::<(), _>("set_weight_version", (weight_version,)).await?;
Ok(())
}
/// Return the committed weight version agreed on by every connected engine.
pub async fn get_weight_version(&self) -> Result<String> {
self.call_utility_consensus("get_weight_version", ()).await
}
/// Return whether the engine is currently sleeping at any level.
pub async fn is_sleeping(&self) -> Result<bool> {
self.call_utility_consensus("is_sleeping", ()).await
@@ -67,6 +67,9 @@ pub fn default_ready_response() -> EngineCoreReadyResponse {
kv_cache_size_tokens: None,
kv_cache_max_concurrency: None,
kv_events_config: None,
weight_transfer_backend: None,
enable_sleep_mode: false,
supports_draft_weight_updates: false,
}
}
@@ -87,6 +87,15 @@ pub struct EngineCoreReadyResponse {
/// KV-event publisher configuration, if configured.
#[serde(default)]
pub kv_events_config: Option<KvEventsConfig>,
/// Configured RL weight-transfer backend, if weight transfer is enabled.
#[serde(default)]
pub weight_transfer_backend: Option<String>,
/// Whether the engine was started with sleep mode enabled.
#[serde(default)]
pub enable_sleep_mode: bool,
/// Whether the engine has a speculative draft model that can be updated.
#[serde(default)]
pub supports_draft_weight_updates: bool,
}
/// Frontend-owned ZMQ addresses that are sent to the engine during startup
@@ -2686,6 +2686,12 @@ fn python_msgpack_fixtures_match_rust_encoding() {
let ready_response: EngineCoreReadyResponse =
rmp_serde::from_slice(&hex::decode(ready_response_hex).unwrap()).unwrap();
assert_eq!(
ready_response.weight_transfer_backend.as_deref(),
Some("nccl")
);
assert!(ready_response.enable_sleep_mode);
assert!(ready_response.supports_draft_weight_updates);
let kv_events_config = ready_response.kv_events_config.expect("KV events config should decode");
assert!(kv_events_config.enable_kv_cache_events);
assert_eq!(kv_events_config.publisher, "zmq");
@@ -412,6 +412,9 @@ class EngineCoreReadyResponse:
kv_cache_size_tokens: int | None = None
kv_cache_max_concurrency: float | None = None
kv_events_config: KVEventsConfig | None = None
weight_transfer_backend: str | None = None
enable_sleep_mode: bool = False
supports_draft_weight_updates: bool = False
ready_response = EngineCoreReadyResponse(
@@ -430,6 +433,9 @@ ready_response = EngineCoreReadyResponse(
max_num_seqs=256,
max_num_batched_tokens=8192,
instance_id="test-instance",
weight_transfer_backend="nccl",
enable_sleep_mode=True,
supports_draft_weight_updates=True,
kv_events_config=KVEventsConfig(
enable_kv_cache_events=True,
publisher="zmq",