diff --git a/rust/src/engine-core-client/examples/external_engine_utility_call.rs b/rust/src/engine-core-client/examples/external_engine_utility_call.rs index ee2a4e57b7..2fff91bc20 100644 --- a/rust/src/engine-core-client/examples/external_engine_utility_call.rs +++ b/rust/src/engine-core-client/examples/external_engine_utility_call.rs @@ -3,6 +3,7 @@ use std::time::Duration; use anyhow::{Context, Result, bail}; use clap::Parser; use tracing_subscriber::EnvFilter; +use vllm_engine_core_client::protocol::utility::PauseMode; use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, TransportMode}; #[derive(Debug, Parser)] @@ -32,8 +33,8 @@ struct Args { reset_external: bool, #[arg(long, default_value_t = 1)] sleep_level: u32, - #[arg(long, default_value = "abort")] - sleep_mode: String, + #[arg(long, default_value_t = PauseMode::Abort)] + sleep_mode: PauseMode, #[arg( long, default_value_t = false, @@ -106,7 +107,7 @@ async fn main() -> Result<()> { if args.skip_sleep_wake { println!("sleep_wake=skipped"); } else { - client.sleep(args.sleep_level, &args.sleep_mode).await.with_context(|| { + client.sleep(args.sleep_level, args.sleep_mode).await.with_context(|| { format!( "failed to call sleep utility with level={} mode={}", args.sleep_level, args.sleep_mode diff --git a/rust/src/engine-core-client/src/client.rs b/rust/src/engine-core-client/src/client.rs index 8f5daba79e..f48bf72cd1 100644 --- a/rust/src/engine-core-client/src/client.rs +++ b/rust/src/engine-core-client/src/client.rs @@ -2,6 +2,7 @@ use std::sync::Arc; use std::time::Duration; use futures::future::{join_all, try_join_all}; +use itertools::Itertools; use serde::Serialize; use tokio::sync::mpsc; use tokio_util::task::AbortOnDropHandle; @@ -12,7 +13,7 @@ use crate::coordinator::CoordinatorHandle; use crate::error::{Error, Result}; use crate::protocol::handshake::EngineCoreReadyResponse; use crate::protocol::lora::LoraRequest; -use crate::protocol::utility::EngineCoreUtilityRequest; +use crate::protocol::utility::{EngineCoreUtilityRequest, PauseMode}; use crate::protocol::{EngineCoreRequest, EngineCoreRequestType, ModelDtype}; use crate::transport::{self, ConnectedEngine}; @@ -571,6 +572,26 @@ impl EngineCoreClient { try_join_all(futures).await } + /// Call a utility method on all connected engines and return the shared + /// result if every engine agrees. + pub async fn call_utility_consensus(&self, method: &str, args: A) -> Result + where + T: serde::de::DeserializeOwned + std::fmt::Debug + PartialEq, + A: serde::Serialize + std::fmt::Debug, + { + let results: Vec = self.call_utility(method, args).await?; + + if results.iter().all_equal() { + // `engine_count >= 1` is enforced during startup handshake so `results` must be non-empty. + Ok(results.into_iter().next().unwrap()) + } else { + Err(Error::InconsistentUtilityResults { + method: method.to_string(), + values: format!("{results:?}"), + }) + } + } + /// Execute `collective_rpc` on all engines and flatten all engine results /// into one list. pub async fn collective_rpc( @@ -599,27 +620,8 @@ impl EngineCoreClient { } /// Return whether the engine is currently sleeping at any level. - /// - /// Under data parallel, all engines should agree on the sleep state: a - /// divergence signals a control-plane bug. Returns - /// `Error::InconsistentUtilityResults` if engines disagree. pub async fn is_sleeping(&self) -> Result { - let results: Vec = self.call_utility("is_sleeping", ()).await?; - // `engine_count >= 1` is enforced during startup handshake, so `results` - // is normally non-empty; fall back to a fail-loud error rather than - // indexing in case that invariant is ever bypassed. - let first = *results.first().ok_or_else(|| Error::InconsistentUtilityResults { - method: "is_sleeping".to_string(), - values: "[]".to_string(), - })?; - if results.iter().all(|&v| v == first) { - Ok(first) - } else { - Err(Error::InconsistentUtilityResults { - method: "is_sleeping".to_string(), - values: format!("{results:?}"), - }) - } + self.call_utility_consensus("is_sleeping", ()).await } /// Reset the multi-modal cache. @@ -643,22 +645,14 @@ impl EngineCoreClient { reset_running_requests: bool, reset_connector: bool, ) -> Result { - let results: Vec = self + Ok(self .call_utility( "reset_prefix_cache", (reset_running_requests, reset_connector), ) - .await?; - // `engine_count >= 1` is enforced during startup handshake, so `results` - // is normally non-empty; fail loud rather than reporting a vacuous - // success (`[].all() == true`) in case that invariant is ever bypassed. - if results.is_empty() { - return Err(Error::InconsistentUtilityResults { - method: "reset_prefix_cache".to_string(), - values: "[]".to_string(), - }); - } - Ok(results.into_iter().all(|ok| ok)) + .await? + .into_iter() + .all(|reset| reset)) } /// Load or refresh one LoRA adapter on every connected engine. @@ -680,7 +674,7 @@ impl EngineCoreClient { } /// Put the engine to sleep. - pub async fn sleep(&self, level: u32, mode: &str) -> Result<()> { + pub async fn sleep(&self, level: u32, mode: PauseMode) -> Result<()> { self.call_utility::<(), _>("sleep", (level, mode)).await?; Ok(()) } @@ -693,7 +687,7 @@ impl EngineCoreClient { } /// Pause the scheduler so generation can be halted - pub async fn pause_scheduler(&self, mode: &str, clear_cache: bool) -> Result<()> { + pub async fn pause_scheduler(&self, mode: PauseMode, clear_cache: bool) -> Result<()> { self.call_utility::<(), _>("pause_scheduler", (mode, clear_cache)).await?; Ok(()) } @@ -706,22 +700,7 @@ impl EngineCoreClient { /// Return whether the scheduler is currently in any pause state. pub async fn is_scheduler_paused(&self) -> Result { - let results: Vec = self.call_utility("is_scheduler_paused", ()).await?; - // `engine_count >= 1` is enforced during startup handshake, so `results` - // is normally non-empty; fall back to a fail-loud error rather than - // indexing in case that invariant is ever bypassed. - let first = *results.first().ok_or_else(|| Error::InconsistentUtilityResults { - method: "is_scheduler_paused".to_string(), - values: "[]".to_string(), - })?; - if results.iter().all(|&v| v == first) { - Ok(first) - } else { - Err(Error::InconsistentUtilityResults { - method: "is_scheduler_paused".to_string(), - values: format!("{results:?}"), - }) - } + self.call_utility_consensus("is_scheduler_paused", ()).await } /// Shut down local client tasks and close transport state. diff --git a/rust/src/engine-core-client/src/protocol/utility.rs b/rust/src/engine-core-client/src/protocol/utility.rs index ef7e862d51..bfaf2736e0 100644 --- a/rust/src/engine-core-client/src/protocol/utility.rs +++ b/rust/src/engine-core-client/src/protocol/utility.rs @@ -1,15 +1,63 @@ use std::any::type_name; -use std::fmt; +use std::{fmt, str::FromStr}; use rmpv::Value; use serde::{Deserialize, Serialize}; use serde_default::DefaultFromSerde; use serde_tuple::{Deserialize_tuple, Serialize_tuple}; +use serde_with::{DeserializeFromStr, SerializeDisplay}; use thiserror_ext::AsReport; use super::{OpaqueValue, default_opaque_value_nil}; use crate::error::{Error, Result}; +/// How pause/sleep utility calls handle in-flight requests. +/// +/// Use display/from-str serde so MessagePack utility args stay as Python +/// literal strings instead of serde enum variant tuples. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, SerializeDisplay, DeserializeFromStr)] +pub enum PauseMode { + /// Abort all in-flight requests immediately. + #[default] + Abort, + /// Wait for in-flight requests to complete. + Wait, + /// Freeze queued requests so they can resume later. + Keep, +} + +impl PauseMode { + /// Return the Python literal used on the utility-call wire. + pub fn as_str(self) -> &'static str { + match self { + Self::Abort => "abort", + Self::Wait => "wait", + Self::Keep => "keep", + } + } +} + +impl fmt::Display for PauseMode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +impl FromStr for PauseMode { + type Err = String; + + fn from_str(value: &str) -> std::result::Result { + match value { + "abort" => Ok(Self::Abort), + "wait" => Ok(Self::Wait), + "keep" => Ok(Self::Keep), + other => Err(format!( + "invalid pause mode `{other}`; expected one of: abort, wait, keep" + )), + } + } +} + /// Utility call id as carried on the engine-core MessagePack wire. /// /// Python emits utility ids as MessagePack integers, including values that may @@ -212,7 +260,7 @@ mod tests { use rmpv::Value; use serde::Serialize; - use super::{EngineCoreUtilityRequest, UtilityOutput, UtilityResultEnvelope}; + use super::{EngineCoreUtilityRequest, PauseMode, UtilityOutput, UtilityResultEnvelope}; use crate::Error; use crate::protocol::{decode_msgpack, decode_value, encode_msgpack}; @@ -241,6 +289,26 @@ mod tests { assert_eq!(array[3], Value::Array(Vec::new())); } + #[test] + fn pause_mode_serializes_as_python_literal() { + let request = + EngineCoreUtilityRequest::new(7, 42, "pause_scheduler", (PauseMode::Abort, true)) + .unwrap(); + + let encoded = encode_msgpack(&request).unwrap(); + let value = decode_value(&encoded).unwrap(); + let array = match value { + Value::Array(array) => array, + other => panic!("expected utility request array, got {other:?}"), + }; + + assert_eq!(array[2], Value::from("pause_scheduler")); + assert_eq!( + array[3], + Value::Array(vec![Value::from("abort"), Value::from(true)]) + ); + } + #[test] fn utility_output_decodes_typed_result() { let output = UtilityOutput { diff --git a/rust/src/server/src/routes/pause.rs b/rust/src/server/src/routes/pause.rs index ef55a6f5e4..934846054e 100644 --- a/rust/src/server/src/routes/pause.rs +++ b/rust/src/server/src/routes/pause.rs @@ -1,8 +1,10 @@ use std::sync::Arc; use axum::Json; +use axum::extract::rejection::QueryRejection; use axum::extract::{Query, State}; use serde::{Deserialize, Serialize}; +use vllm_engine_core_client::protocol::utility::PauseMode; use crate::error::ApiError; use crate::state::AppState; @@ -10,8 +12,8 @@ use crate::utils::utility_call_error; #[derive(Debug, Deserialize)] pub(crate) struct PauseParams { - #[serde(default = "default_pause_mode")] - mode: String, + #[serde(default)] + mode: PauseMode, #[serde(default = "default_clear_cache")] clear_cache: bool, } @@ -26,17 +28,14 @@ pub(crate) struct IsPausedResponse { is_paused: bool, } -/// Pause modes accepted by the engine (`PauseMode` in the Python frontend). -const VALID_PAUSE_MODES: [&str; 3] = ["abort", "wait", "keep"]; - -fn default_pause_mode() -> String { - "abort".to_string() -} - const fn default_clear_cache() -> bool { true } +fn invalid_query(error: QueryRejection) -> ApiError { + ApiError::invalid_request(error.body_text(), Some("mode")) +} + // TODO: the Python frontend also accepts the deprecated // `wait_for_inflight_requests` flag (equivalent to `mode="wait"`); it is // intentionally omitted here in favor of the `mode` parameter. @@ -44,21 +43,13 @@ const fn default_clear_cache() -> bool { /// Pause the scheduler so generation can be halted (e.g. for weight updates). pub async fn pause( State(state): State>, - Query(params): Query, + params: Result, QueryRejection>, ) -> Result, ApiError> { - if !VALID_PAUSE_MODES.contains(¶ms.mode.as_str()) { - return Err(ApiError::invalid_request( - format!( - "Invalid pause mode '{}'; expected one of: abort, wait, keep", - params.mode - ), - Some("mode"), - )); - } + let Query(params) = params.map_err(invalid_query)?; state .engine_core_client() - .pause_scheduler(¶ms.mode, params.clear_cache) + .pause_scheduler(params.mode, params.clear_cache) .await .map_err(|error| utility_call_error("pause", error))?; diff --git a/rust/src/server/src/routes/sleep.rs b/rust/src/server/src/routes/sleep.rs index d7b279699b..9df2e31e76 100644 --- a/rust/src/server/src/routes/sleep.rs +++ b/rust/src/server/src/routes/sleep.rs @@ -1,9 +1,11 @@ use std::sync::Arc; use axum::Json; +use axum::extract::rejection::QueryRejection; use axum::extract::{Query, State}; use axum::http::StatusCode; use serde::{Deserialize, Serialize}; +use vllm_engine_core_client::protocol::utility::PauseMode; use crate::error::ApiError; use crate::state::AppState; @@ -18,8 +20,8 @@ pub(crate) struct IsSleepingResponse { pub(crate) struct SleepParams { #[serde(default = "default_sleep_level")] level: u32, - #[serde(default = "default_sleep_mode")] - mode: String, + #[serde(default)] + mode: PauseMode, } #[derive(Debug, Default, Deserialize)] @@ -32,18 +34,20 @@ const fn default_sleep_level() -> u32 { 1 } -fn default_sleep_mode() -> String { - "abort".to_string() +fn invalid_query(error: QueryRejection) -> ApiError { + ApiError::invalid_request(error.body_text(), Some("mode")) } /// Put the engine to sleep. pub async fn sleep( State(state): State>, - Query(params): Query, + params: Result, QueryRejection>, ) -> Result { + let Query(params) = params.map_err(invalid_query)?; + state .engine_core_client() - .sleep(params.level, ¶ms.mode) + .sleep(params.level, params.mode) .await .map_err(|error| utility_call_error("sleep", error))?;