[Rust Frontend] [Refactor] Refine utility call interfaces (#44856)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-06-08 15:13:08 -07:00
committed by GitHub
parent 6afa25000c
commit bc941f375d
5 changed files with 125 additions and 82 deletions
@@ -3,6 +3,7 @@ use std::time::Duration;
use anyhow::{Context, Result, bail};
use clap::Parser;
use tracing_subscriber::EnvFilter;
use vllm_engine_core_client::protocol::utility::PauseMode;
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, TransportMode};
#[derive(Debug, Parser)]
@@ -32,8 +33,8 @@ struct Args {
reset_external: bool,
#[arg(long, default_value_t = 1)]
sleep_level: u32,
#[arg(long, default_value = "abort")]
sleep_mode: String,
#[arg(long, default_value_t = PauseMode::Abort)]
sleep_mode: PauseMode,
#[arg(
long,
default_value_t = false,
@@ -106,7 +107,7 @@ async fn main() -> Result<()> {
if args.skip_sleep_wake {
println!("sleep_wake=skipped");
} else {
client.sleep(args.sleep_level, &args.sleep_mode).await.with_context(|| {
client.sleep(args.sleep_level, args.sleep_mode).await.with_context(|| {
format!(
"failed to call sleep utility with level={} mode={}",
args.sleep_level, args.sleep_mode
+30 -51
View File
@@ -2,6 +2,7 @@ use std::sync::Arc;
use std::time::Duration;
use futures::future::{join_all, try_join_all};
use itertools::Itertools;
use serde::Serialize;
use tokio::sync::mpsc;
use tokio_util::task::AbortOnDropHandle;
@@ -12,7 +13,7 @@ use crate::coordinator::CoordinatorHandle;
use crate::error::{Error, Result};
use crate::protocol::handshake::EngineCoreReadyResponse;
use crate::protocol::lora::LoraRequest;
use crate::protocol::utility::EngineCoreUtilityRequest;
use crate::protocol::utility::{EngineCoreUtilityRequest, PauseMode};
use crate::protocol::{EngineCoreRequest, EngineCoreRequestType, ModelDtype};
use crate::transport::{self, ConnectedEngine};
@@ -571,6 +572,26 @@ impl EngineCoreClient {
try_join_all(futures).await
}
/// Call a utility method on all connected engines and return the shared
/// result if every engine agrees.
pub async fn call_utility_consensus<T, A>(&self, method: &str, args: A) -> Result<T>
where
T: serde::de::DeserializeOwned + std::fmt::Debug + PartialEq,
A: serde::Serialize + std::fmt::Debug,
{
let results: Vec<T> = self.call_utility(method, args).await?;
if results.iter().all_equal() {
// `engine_count >= 1` is enforced during startup handshake so `results` must be non-empty.
Ok(results.into_iter().next().unwrap())
} else {
Err(Error::InconsistentUtilityResults {
method: method.to_string(),
values: format!("{results:?}"),
})
}
}
/// Execute `collective_rpc` on all engines and flatten all engine results
/// into one list.
pub async fn collective_rpc<A, K>(
@@ -599,27 +620,8 @@ impl EngineCoreClient {
}
/// Return whether the engine is currently sleeping at any level.
///
/// Under data parallel, all engines should agree on the sleep state: a
/// divergence signals a control-plane bug. Returns
/// `Error::InconsistentUtilityResults` if engines disagree.
pub async fn is_sleeping(&self) -> Result<bool> {
let results: Vec<bool> = self.call_utility("is_sleeping", ()).await?;
// `engine_count >= 1` is enforced during startup handshake, so `results`
// is normally non-empty; fall back to a fail-loud error rather than
// indexing in case that invariant is ever bypassed.
let first = *results.first().ok_or_else(|| Error::InconsistentUtilityResults {
method: "is_sleeping".to_string(),
values: "[]".to_string(),
})?;
if results.iter().all(|&v| v == first) {
Ok(first)
} else {
Err(Error::InconsistentUtilityResults {
method: "is_sleeping".to_string(),
values: format!("{results:?}"),
})
}
self.call_utility_consensus("is_sleeping", ()).await
}
/// Reset the multi-modal cache.
@@ -643,22 +645,14 @@ impl EngineCoreClient {
reset_running_requests: bool,
reset_connector: bool,
) -> Result<bool> {
let results: Vec<bool> = self
Ok(self
.call_utility(
"reset_prefix_cache",
(reset_running_requests, reset_connector),
)
.await?;
// `engine_count >= 1` is enforced during startup handshake, so `results`
// is normally non-empty; fail loud rather than reporting a vacuous
// success (`[].all() == true`) in case that invariant is ever bypassed.
if results.is_empty() {
return Err(Error::InconsistentUtilityResults {
method: "reset_prefix_cache".to_string(),
values: "[]".to_string(),
});
}
Ok(results.into_iter().all(|ok| ok))
.await?
.into_iter()
.all(|reset| reset))
}
/// Load or refresh one LoRA adapter on every connected engine.
@@ -680,7 +674,7 @@ impl EngineCoreClient {
}
/// Put the engine to sleep.
pub async fn sleep(&self, level: u32, mode: &str) -> Result<()> {
pub async fn sleep(&self, level: u32, mode: PauseMode) -> Result<()> {
self.call_utility::<(), _>("sleep", (level, mode)).await?;
Ok(())
}
@@ -693,7 +687,7 @@ impl EngineCoreClient {
}
/// Pause the scheduler so generation can be halted
pub async fn pause_scheduler(&self, mode: &str, clear_cache: bool) -> Result<()> {
pub async fn pause_scheduler(&self, mode: PauseMode, clear_cache: bool) -> Result<()> {
self.call_utility::<(), _>("pause_scheduler", (mode, clear_cache)).await?;
Ok(())
}
@@ -706,22 +700,7 @@ impl EngineCoreClient {
/// Return whether the scheduler is currently in any pause state.
pub async fn is_scheduler_paused(&self) -> Result<bool> {
let results: Vec<bool> = self.call_utility("is_scheduler_paused", ()).await?;
// `engine_count >= 1` is enforced during startup handshake, so `results`
// is normally non-empty; fall back to a fail-loud error rather than
// indexing in case that invariant is ever bypassed.
let first = *results.first().ok_or_else(|| Error::InconsistentUtilityResults {
method: "is_scheduler_paused".to_string(),
values: "[]".to_string(),
})?;
if results.iter().all(|&v| v == first) {
Ok(first)
} else {
Err(Error::InconsistentUtilityResults {
method: "is_scheduler_paused".to_string(),
values: format!("{results:?}"),
})
}
self.call_utility_consensus("is_scheduler_paused", ()).await
}
/// Shut down local client tasks and close transport state.
@@ -1,15 +1,63 @@
use std::any::type_name;
use std::fmt;
use std::{fmt, str::FromStr};
use rmpv::Value;
use serde::{Deserialize, Serialize};
use serde_default::DefaultFromSerde;
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
use serde_with::{DeserializeFromStr, SerializeDisplay};
use thiserror_ext::AsReport;
use super::{OpaqueValue, default_opaque_value_nil};
use crate::error::{Error, Result};
/// How pause/sleep utility calls handle in-flight requests.
///
/// Use display/from-str serde so MessagePack utility args stay as Python
/// literal strings instead of serde enum variant tuples.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, SerializeDisplay, DeserializeFromStr)]
pub enum PauseMode {
/// Abort all in-flight requests immediately.
#[default]
Abort,
/// Wait for in-flight requests to complete.
Wait,
/// Freeze queued requests so they can resume later.
Keep,
}
impl PauseMode {
/// Return the Python literal used on the utility-call wire.
pub fn as_str(self) -> &'static str {
match self {
Self::Abort => "abort",
Self::Wait => "wait",
Self::Keep => "keep",
}
}
}
impl fmt::Display for PauseMode {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.as_str())
}
}
impl FromStr for PauseMode {
type Err = String;
fn from_str(value: &str) -> std::result::Result<Self, Self::Err> {
match value {
"abort" => Ok(Self::Abort),
"wait" => Ok(Self::Wait),
"keep" => Ok(Self::Keep),
other => Err(format!(
"invalid pause mode `{other}`; expected one of: abort, wait, keep"
)),
}
}
}
/// Utility call id as carried on the engine-core MessagePack wire.
///
/// Python emits utility ids as MessagePack integers, including values that may
@@ -212,7 +260,7 @@ mod tests {
use rmpv::Value;
use serde::Serialize;
use super::{EngineCoreUtilityRequest, UtilityOutput, UtilityResultEnvelope};
use super::{EngineCoreUtilityRequest, PauseMode, UtilityOutput, UtilityResultEnvelope};
use crate::Error;
use crate::protocol::{decode_msgpack, decode_value, encode_msgpack};
@@ -241,6 +289,26 @@ mod tests {
assert_eq!(array[3], Value::Array(Vec::new()));
}
#[test]
fn pause_mode_serializes_as_python_literal() {
let request =
EngineCoreUtilityRequest::new(7, 42, "pause_scheduler", (PauseMode::Abort, true))
.unwrap();
let encoded = encode_msgpack(&request).unwrap();
let value = decode_value(&encoded).unwrap();
let array = match value {
Value::Array(array) => array,
other => panic!("expected utility request array, got {other:?}"),
};
assert_eq!(array[2], Value::from("pause_scheduler"));
assert_eq!(
array[3],
Value::Array(vec![Value::from("abort"), Value::from(true)])
);
}
#[test]
fn utility_output_decodes_typed_result() {
let output = UtilityOutput {
+11 -20
View File
@@ -1,8 +1,10 @@
use std::sync::Arc;
use axum::Json;
use axum::extract::rejection::QueryRejection;
use axum::extract::{Query, State};
use serde::{Deserialize, Serialize};
use vllm_engine_core_client::protocol::utility::PauseMode;
use crate::error::ApiError;
use crate::state::AppState;
@@ -10,8 +12,8 @@ use crate::utils::utility_call_error;
#[derive(Debug, Deserialize)]
pub(crate) struct PauseParams {
#[serde(default = "default_pause_mode")]
mode: String,
#[serde(default)]
mode: PauseMode,
#[serde(default = "default_clear_cache")]
clear_cache: bool,
}
@@ -26,17 +28,14 @@ pub(crate) struct IsPausedResponse {
is_paused: bool,
}
/// Pause modes accepted by the engine (`PauseMode` in the Python frontend).
const VALID_PAUSE_MODES: [&str; 3] = ["abort", "wait", "keep"];
fn default_pause_mode() -> String {
"abort".to_string()
}
const fn default_clear_cache() -> bool {
true
}
fn invalid_query(error: QueryRejection) -> ApiError {
ApiError::invalid_request(error.body_text(), Some("mode"))
}
// TODO: the Python frontend also accepts the deprecated
// `wait_for_inflight_requests` flag (equivalent to `mode="wait"`); it is
// intentionally omitted here in favor of the `mode` parameter.
@@ -44,21 +43,13 @@ const fn default_clear_cache() -> bool {
/// Pause the scheduler so generation can be halted (e.g. for weight updates).
pub async fn pause(
State(state): State<Arc<AppState>>,
Query(params): Query<PauseParams>,
params: Result<Query<PauseParams>, QueryRejection>,
) -> Result<Json<StatusResponse>, ApiError> {
if !VALID_PAUSE_MODES.contains(&params.mode.as_str()) {
return Err(ApiError::invalid_request(
format!(
"Invalid pause mode '{}'; expected one of: abort, wait, keep",
params.mode
),
Some("mode"),
));
}
let Query(params) = params.map_err(invalid_query)?;
state
.engine_core_client()
.pause_scheduler(&params.mode, params.clear_cache)
.pause_scheduler(params.mode, params.clear_cache)
.await
.map_err(|error| utility_call_error("pause", error))?;
+10 -6
View File
@@ -1,9 +1,11 @@
use std::sync::Arc;
use axum::Json;
use axum::extract::rejection::QueryRejection;
use axum::extract::{Query, State};
use axum::http::StatusCode;
use serde::{Deserialize, Serialize};
use vllm_engine_core_client::protocol::utility::PauseMode;
use crate::error::ApiError;
use crate::state::AppState;
@@ -18,8 +20,8 @@ pub(crate) struct IsSleepingResponse {
pub(crate) struct SleepParams {
#[serde(default = "default_sleep_level")]
level: u32,
#[serde(default = "default_sleep_mode")]
mode: String,
#[serde(default)]
mode: PauseMode,
}
#[derive(Debug, Default, Deserialize)]
@@ -32,18 +34,20 @@ const fn default_sleep_level() -> u32 {
1
}
fn default_sleep_mode() -> String {
"abort".to_string()
fn invalid_query(error: QueryRejection) -> ApiError {
ApiError::invalid_request(error.body_text(), Some("mode"))
}
/// Put the engine to sleep.
pub async fn sleep(
State(state): State<Arc<AppState>>,
Query(params): Query<SleepParams>,
params: Result<Query<SleepParams>, QueryRejection>,
) -> Result<StatusCode, ApiError> {
let Query(params) = params.map_err(invalid_query)?;
state
.engine_core_client()
.sleep(params.level, &params.mode)
.sleep(params.level, params.mode)
.await
.map_err(|error| utility_call_error("sleep", error))?;