[Rust Frontend][gRPC] Add RL lifecycle control (#51316)
Signed-off-by: Connor Carpenter <connorc@nvidia.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Nick Hill
parent
3e3ceb1961
commit
f473870ecf
@@ -1,12 +1,14 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use futures::future::{join_all, try_join_all};
|
||||
use itertools::Itertools;
|
||||
use serde::Serialize;
|
||||
use serde_json::Value as JsonValue;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_util::task::AbortOnDropHandle;
|
||||
use tracing::{debug, info, trace};
|
||||
@@ -674,6 +676,77 @@ impl EngineCoreClient {
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Initialize the configured RL weight-transfer backend.
|
||||
pub async fn init_weight_transfer_engine(&self, init_info: JsonValue) -> Result<()> {
|
||||
self.collective_rpc(
|
||||
"init_weight_transfer_engine",
|
||||
None,
|
||||
Vec::<JsonValue>::new(),
|
||||
BTreeMap::from([("init_info".to_string(), init_info)]),
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Start a weight update for the base model.
|
||||
pub async fn start_weight_update(&self) -> Result<()> {
|
||||
self.collective_rpc(
|
||||
"start_weight_update",
|
||||
None,
|
||||
Vec::<JsonValue>::new(),
|
||||
BTreeMap::<String, JsonValue>::new(),
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Start a weight update for the speculative draft model.
|
||||
pub async fn start_draft_weight_update(&self) -> Result<()> {
|
||||
self.collective_rpc(
|
||||
"start_draft_weight_update",
|
||||
None,
|
||||
Vec::<JsonValue>::new(),
|
||||
BTreeMap::<String, JsonValue>::new(),
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Apply one backend-specific weight metadata chunk.
|
||||
pub async fn update_weights(&self, update_info: JsonValue) -> Result<()> {
|
||||
self.collective_rpc(
|
||||
"update_weights",
|
||||
None,
|
||||
Vec::<JsonValue>::new(),
|
||||
BTreeMap::from([("update_info".to_string(), update_info)]),
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Finish the current weight update.
|
||||
pub async fn finish_weight_update(&self) -> Result<()> {
|
||||
self.collective_rpc(
|
||||
"finish_weight_update",
|
||||
None,
|
||||
Vec::<JsonValue>::new(),
|
||||
BTreeMap::<String, JsonValue>::new(),
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Set the committed weight version on every connected engine.
|
||||
pub async fn set_weight_version(&self, weight_version: &str) -> Result<()> {
|
||||
self.call_utility::<(), _>("set_weight_version", (weight_version,)).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Return the committed weight version agreed on by every connected engine.
|
||||
pub async fn get_weight_version(&self) -> Result<String> {
|
||||
self.call_utility_consensus("get_weight_version", ()).await
|
||||
}
|
||||
|
||||
/// Return whether the engine is currently sleeping at any level.
|
||||
pub async fn is_sleeping(&self) -> Result<bool> {
|
||||
self.call_utility_consensus("is_sleeping", ()).await
|
||||
|
||||
@@ -67,6 +67,9 @@ pub fn default_ready_response() -> EngineCoreReadyResponse {
|
||||
kv_cache_size_tokens: None,
|
||||
kv_cache_max_concurrency: None,
|
||||
kv_events_config: None,
|
||||
weight_transfer_backend: None,
|
||||
enable_sleep_mode: false,
|
||||
supports_draft_weight_updates: false,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -87,6 +87,15 @@ pub struct EngineCoreReadyResponse {
|
||||
/// KV-event publisher configuration, if configured.
|
||||
#[serde(default)]
|
||||
pub kv_events_config: Option<KvEventsConfig>,
|
||||
/// Configured RL weight-transfer backend, if weight transfer is enabled.
|
||||
#[serde(default)]
|
||||
pub weight_transfer_backend: Option<String>,
|
||||
/// Whether the engine was started with sleep mode enabled.
|
||||
#[serde(default)]
|
||||
pub enable_sleep_mode: bool,
|
||||
/// Whether the engine has a speculative draft model that can be updated.
|
||||
#[serde(default)]
|
||||
pub supports_draft_weight_updates: bool,
|
||||
}
|
||||
|
||||
/// Frontend-owned ZMQ addresses that are sent to the engine during startup
|
||||
|
||||
@@ -2686,6 +2686,12 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
|
||||
let ready_response: EngineCoreReadyResponse =
|
||||
rmp_serde::from_slice(&hex::decode(ready_response_hex).unwrap()).unwrap();
|
||||
assert_eq!(
|
||||
ready_response.weight_transfer_backend.as_deref(),
|
||||
Some("nccl")
|
||||
);
|
||||
assert!(ready_response.enable_sleep_mode);
|
||||
assert!(ready_response.supports_draft_weight_updates);
|
||||
let kv_events_config = ready_response.kv_events_config.expect("KV events config should decode");
|
||||
assert!(kv_events_config.enable_kv_cache_events);
|
||||
assert_eq!(kv_events_config.publisher, "zmq");
|
||||
|
||||
@@ -412,6 +412,9 @@ class EngineCoreReadyResponse:
|
||||
kv_cache_size_tokens: int | None = None
|
||||
kv_cache_max_concurrency: float | None = None
|
||||
kv_events_config: KVEventsConfig | None = None
|
||||
weight_transfer_backend: str | None = None
|
||||
enable_sleep_mode: bool = False
|
||||
supports_draft_weight_updates: bool = False
|
||||
|
||||
|
||||
ready_response = EngineCoreReadyResponse(
|
||||
@@ -430,6 +433,9 @@ ready_response = EngineCoreReadyResponse(
|
||||
max_num_seqs=256,
|
||||
max_num_batched_tokens=8192,
|
||||
instance_id="test-instance",
|
||||
weight_transfer_backend="nccl",
|
||||
enable_sleep_mode=True,
|
||||
supports_draft_weight_updates=True,
|
||||
kv_events_config=KVEventsConfig(
|
||||
enable_kv_cache_events=True,
|
||||
publisher="zmq",
|
||||
|
||||
Reference in New Issue
Block a user