[Rust Frontend] [Refactor] Refine utility call interfaces (#44856)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -3,6 +3,7 @@ use std::time::Duration;
|
||||
use anyhow::{Context, Result, bail};
|
||||
use clap::Parser;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use vllm_engine_core_client::protocol::utility::PauseMode;
|
||||
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, TransportMode};
|
||||
|
||||
#[derive(Debug, Parser)]
|
||||
@@ -32,8 +33,8 @@ struct Args {
|
||||
reset_external: bool,
|
||||
#[arg(long, default_value_t = 1)]
|
||||
sleep_level: u32,
|
||||
#[arg(long, default_value = "abort")]
|
||||
sleep_mode: String,
|
||||
#[arg(long, default_value_t = PauseMode::Abort)]
|
||||
sleep_mode: PauseMode,
|
||||
#[arg(
|
||||
long,
|
||||
default_value_t = false,
|
||||
@@ -106,7 +107,7 @@ async fn main() -> Result<()> {
|
||||
if args.skip_sleep_wake {
|
||||
println!("sleep_wake=skipped");
|
||||
} else {
|
||||
client.sleep(args.sleep_level, &args.sleep_mode).await.with_context(|| {
|
||||
client.sleep(args.sleep_level, args.sleep_mode).await.with_context(|| {
|
||||
format!(
|
||||
"failed to call sleep utility with level={} mode={}",
|
||||
args.sleep_level, args.sleep_mode
|
||||
|
||||
@@ -2,6 +2,7 @@ use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use futures::future::{join_all, try_join_all};
|
||||
use itertools::Itertools;
|
||||
use serde::Serialize;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_util::task::AbortOnDropHandle;
|
||||
@@ -12,7 +13,7 @@ use crate::coordinator::CoordinatorHandle;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::protocol::handshake::EngineCoreReadyResponse;
|
||||
use crate::protocol::lora::LoraRequest;
|
||||
use crate::protocol::utility::EngineCoreUtilityRequest;
|
||||
use crate::protocol::utility::{EngineCoreUtilityRequest, PauseMode};
|
||||
use crate::protocol::{EngineCoreRequest, EngineCoreRequestType, ModelDtype};
|
||||
use crate::transport::{self, ConnectedEngine};
|
||||
|
||||
@@ -571,6 +572,26 @@ impl EngineCoreClient {
|
||||
try_join_all(futures).await
|
||||
}
|
||||
|
||||
/// Call a utility method on all connected engines and return the shared
|
||||
/// result if every engine agrees.
|
||||
pub async fn call_utility_consensus<T, A>(&self, method: &str, args: A) -> Result<T>
|
||||
where
|
||||
T: serde::de::DeserializeOwned + std::fmt::Debug + PartialEq,
|
||||
A: serde::Serialize + std::fmt::Debug,
|
||||
{
|
||||
let results: Vec<T> = self.call_utility(method, args).await?;
|
||||
|
||||
if results.iter().all_equal() {
|
||||
// `engine_count >= 1` is enforced during startup handshake so `results` must be non-empty.
|
||||
Ok(results.into_iter().next().unwrap())
|
||||
} else {
|
||||
Err(Error::InconsistentUtilityResults {
|
||||
method: method.to_string(),
|
||||
values: format!("{results:?}"),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Execute `collective_rpc` on all engines and flatten all engine results
|
||||
/// into one list.
|
||||
pub async fn collective_rpc<A, K>(
|
||||
@@ -599,27 +620,8 @@ impl EngineCoreClient {
|
||||
}
|
||||
|
||||
/// Return whether the engine is currently sleeping at any level.
|
||||
///
|
||||
/// Under data parallel, all engines should agree on the sleep state: a
|
||||
/// divergence signals a control-plane bug. Returns
|
||||
/// `Error::InconsistentUtilityResults` if engines disagree.
|
||||
pub async fn is_sleeping(&self) -> Result<bool> {
|
||||
let results: Vec<bool> = self.call_utility("is_sleeping", ()).await?;
|
||||
// `engine_count >= 1` is enforced during startup handshake, so `results`
|
||||
// is normally non-empty; fall back to a fail-loud error rather than
|
||||
// indexing in case that invariant is ever bypassed.
|
||||
let first = *results.first().ok_or_else(|| Error::InconsistentUtilityResults {
|
||||
method: "is_sleeping".to_string(),
|
||||
values: "[]".to_string(),
|
||||
})?;
|
||||
if results.iter().all(|&v| v == first) {
|
||||
Ok(first)
|
||||
} else {
|
||||
Err(Error::InconsistentUtilityResults {
|
||||
method: "is_sleeping".to_string(),
|
||||
values: format!("{results:?}"),
|
||||
})
|
||||
}
|
||||
self.call_utility_consensus("is_sleeping", ()).await
|
||||
}
|
||||
|
||||
/// Reset the multi-modal cache.
|
||||
@@ -643,22 +645,14 @@ impl EngineCoreClient {
|
||||
reset_running_requests: bool,
|
||||
reset_connector: bool,
|
||||
) -> Result<bool> {
|
||||
let results: Vec<bool> = self
|
||||
Ok(self
|
||||
.call_utility(
|
||||
"reset_prefix_cache",
|
||||
(reset_running_requests, reset_connector),
|
||||
)
|
||||
.await?;
|
||||
// `engine_count >= 1` is enforced during startup handshake, so `results`
|
||||
// is normally non-empty; fail loud rather than reporting a vacuous
|
||||
// success (`[].all() == true`) in case that invariant is ever bypassed.
|
||||
if results.is_empty() {
|
||||
return Err(Error::InconsistentUtilityResults {
|
||||
method: "reset_prefix_cache".to_string(),
|
||||
values: "[]".to_string(),
|
||||
});
|
||||
}
|
||||
Ok(results.into_iter().all(|ok| ok))
|
||||
.await?
|
||||
.into_iter()
|
||||
.all(|reset| reset))
|
||||
}
|
||||
|
||||
/// Load or refresh one LoRA adapter on every connected engine.
|
||||
@@ -680,7 +674,7 @@ impl EngineCoreClient {
|
||||
}
|
||||
|
||||
/// Put the engine to sleep.
|
||||
pub async fn sleep(&self, level: u32, mode: &str) -> Result<()> {
|
||||
pub async fn sleep(&self, level: u32, mode: PauseMode) -> Result<()> {
|
||||
self.call_utility::<(), _>("sleep", (level, mode)).await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -693,7 +687,7 @@ impl EngineCoreClient {
|
||||
}
|
||||
|
||||
/// Pause the scheduler so generation can be halted
|
||||
pub async fn pause_scheduler(&self, mode: &str, clear_cache: bool) -> Result<()> {
|
||||
pub async fn pause_scheduler(&self, mode: PauseMode, clear_cache: bool) -> Result<()> {
|
||||
self.call_utility::<(), _>("pause_scheduler", (mode, clear_cache)).await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -706,22 +700,7 @@ impl EngineCoreClient {
|
||||
|
||||
/// Return whether the scheduler is currently in any pause state.
|
||||
pub async fn is_scheduler_paused(&self) -> Result<bool> {
|
||||
let results: Vec<bool> = self.call_utility("is_scheduler_paused", ()).await?;
|
||||
// `engine_count >= 1` is enforced during startup handshake, so `results`
|
||||
// is normally non-empty; fall back to a fail-loud error rather than
|
||||
// indexing in case that invariant is ever bypassed.
|
||||
let first = *results.first().ok_or_else(|| Error::InconsistentUtilityResults {
|
||||
method: "is_scheduler_paused".to_string(),
|
||||
values: "[]".to_string(),
|
||||
})?;
|
||||
if results.iter().all(|&v| v == first) {
|
||||
Ok(first)
|
||||
} else {
|
||||
Err(Error::InconsistentUtilityResults {
|
||||
method: "is_scheduler_paused".to_string(),
|
||||
values: format!("{results:?}"),
|
||||
})
|
||||
}
|
||||
self.call_utility_consensus("is_scheduler_paused", ()).await
|
||||
}
|
||||
|
||||
/// Shut down local client tasks and close transport state.
|
||||
|
||||
@@ -1,15 +1,63 @@
|
||||
use std::any::type_name;
|
||||
use std::fmt;
|
||||
use std::{fmt, str::FromStr};
|
||||
|
||||
use rmpv::Value;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_default::DefaultFromSerde;
|
||||
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
|
||||
use serde_with::{DeserializeFromStr, SerializeDisplay};
|
||||
use thiserror_ext::AsReport;
|
||||
|
||||
use super::{OpaqueValue, default_opaque_value_nil};
|
||||
use crate::error::{Error, Result};
|
||||
|
||||
/// How pause/sleep utility calls handle in-flight requests.
|
||||
///
|
||||
/// Use display/from-str serde so MessagePack utility args stay as Python
|
||||
/// literal strings instead of serde enum variant tuples.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, SerializeDisplay, DeserializeFromStr)]
|
||||
pub enum PauseMode {
|
||||
/// Abort all in-flight requests immediately.
|
||||
#[default]
|
||||
Abort,
|
||||
/// Wait for in-flight requests to complete.
|
||||
Wait,
|
||||
/// Freeze queued requests so they can resume later.
|
||||
Keep,
|
||||
}
|
||||
|
||||
impl PauseMode {
|
||||
/// Return the Python literal used on the utility-call wire.
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Abort => "abort",
|
||||
Self::Wait => "wait",
|
||||
Self::Keep => "keep",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for PauseMode {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for PauseMode {
|
||||
type Err = String;
|
||||
|
||||
fn from_str(value: &str) -> std::result::Result<Self, Self::Err> {
|
||||
match value {
|
||||
"abort" => Ok(Self::Abort),
|
||||
"wait" => Ok(Self::Wait),
|
||||
"keep" => Ok(Self::Keep),
|
||||
other => Err(format!(
|
||||
"invalid pause mode `{other}`; expected one of: abort, wait, keep"
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Utility call id as carried on the engine-core MessagePack wire.
|
||||
///
|
||||
/// Python emits utility ids as MessagePack integers, including values that may
|
||||
@@ -212,7 +260,7 @@ mod tests {
|
||||
use rmpv::Value;
|
||||
use serde::Serialize;
|
||||
|
||||
use super::{EngineCoreUtilityRequest, UtilityOutput, UtilityResultEnvelope};
|
||||
use super::{EngineCoreUtilityRequest, PauseMode, UtilityOutput, UtilityResultEnvelope};
|
||||
use crate::Error;
|
||||
use crate::protocol::{decode_msgpack, decode_value, encode_msgpack};
|
||||
|
||||
@@ -241,6 +289,26 @@ mod tests {
|
||||
assert_eq!(array[3], Value::Array(Vec::new()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pause_mode_serializes_as_python_literal() {
|
||||
let request =
|
||||
EngineCoreUtilityRequest::new(7, 42, "pause_scheduler", (PauseMode::Abort, true))
|
||||
.unwrap();
|
||||
|
||||
let encoded = encode_msgpack(&request).unwrap();
|
||||
let value = decode_value(&encoded).unwrap();
|
||||
let array = match value {
|
||||
Value::Array(array) => array,
|
||||
other => panic!("expected utility request array, got {other:?}"),
|
||||
};
|
||||
|
||||
assert_eq!(array[2], Value::from("pause_scheduler"));
|
||||
assert_eq!(
|
||||
array[3],
|
||||
Value::Array(vec![Value::from("abort"), Value::from(true)])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn utility_output_decodes_typed_result() {
|
||||
let output = UtilityOutput {
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::Json;
|
||||
use axum::extract::rejection::QueryRejection;
|
||||
use axum::extract::{Query, State};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use vllm_engine_core_client::protocol::utility::PauseMode;
|
||||
|
||||
use crate::error::ApiError;
|
||||
use crate::state::AppState;
|
||||
@@ -10,8 +12,8 @@ use crate::utils::utility_call_error;
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(crate) struct PauseParams {
|
||||
#[serde(default = "default_pause_mode")]
|
||||
mode: String,
|
||||
#[serde(default)]
|
||||
mode: PauseMode,
|
||||
#[serde(default = "default_clear_cache")]
|
||||
clear_cache: bool,
|
||||
}
|
||||
@@ -26,17 +28,14 @@ pub(crate) struct IsPausedResponse {
|
||||
is_paused: bool,
|
||||
}
|
||||
|
||||
/// Pause modes accepted by the engine (`PauseMode` in the Python frontend).
|
||||
const VALID_PAUSE_MODES: [&str; 3] = ["abort", "wait", "keep"];
|
||||
|
||||
fn default_pause_mode() -> String {
|
||||
"abort".to_string()
|
||||
}
|
||||
|
||||
const fn default_clear_cache() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn invalid_query(error: QueryRejection) -> ApiError {
|
||||
ApiError::invalid_request(error.body_text(), Some("mode"))
|
||||
}
|
||||
|
||||
// TODO: the Python frontend also accepts the deprecated
|
||||
// `wait_for_inflight_requests` flag (equivalent to `mode="wait"`); it is
|
||||
// intentionally omitted here in favor of the `mode` parameter.
|
||||
@@ -44,21 +43,13 @@ const fn default_clear_cache() -> bool {
|
||||
/// Pause the scheduler so generation can be halted (e.g. for weight updates).
|
||||
pub async fn pause(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Query(params): Query<PauseParams>,
|
||||
params: Result<Query<PauseParams>, QueryRejection>,
|
||||
) -> Result<Json<StatusResponse>, ApiError> {
|
||||
if !VALID_PAUSE_MODES.contains(¶ms.mode.as_str()) {
|
||||
return Err(ApiError::invalid_request(
|
||||
format!(
|
||||
"Invalid pause mode '{}'; expected one of: abort, wait, keep",
|
||||
params.mode
|
||||
),
|
||||
Some("mode"),
|
||||
));
|
||||
}
|
||||
let Query(params) = params.map_err(invalid_query)?;
|
||||
|
||||
state
|
||||
.engine_core_client()
|
||||
.pause_scheduler(¶ms.mode, params.clear_cache)
|
||||
.pause_scheduler(params.mode, params.clear_cache)
|
||||
.await
|
||||
.map_err(|error| utility_call_error("pause", error))?;
|
||||
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::Json;
|
||||
use axum::extract::rejection::QueryRejection;
|
||||
use axum::extract::{Query, State};
|
||||
use axum::http::StatusCode;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use vllm_engine_core_client::protocol::utility::PauseMode;
|
||||
|
||||
use crate::error::ApiError;
|
||||
use crate::state::AppState;
|
||||
@@ -18,8 +20,8 @@ pub(crate) struct IsSleepingResponse {
|
||||
pub(crate) struct SleepParams {
|
||||
#[serde(default = "default_sleep_level")]
|
||||
level: u32,
|
||||
#[serde(default = "default_sleep_mode")]
|
||||
mode: String,
|
||||
#[serde(default)]
|
||||
mode: PauseMode,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
@@ -32,18 +34,20 @@ const fn default_sleep_level() -> u32 {
|
||||
1
|
||||
}
|
||||
|
||||
fn default_sleep_mode() -> String {
|
||||
"abort".to_string()
|
||||
fn invalid_query(error: QueryRejection) -> ApiError {
|
||||
ApiError::invalid_request(error.body_text(), Some("mode"))
|
||||
}
|
||||
|
||||
/// Put the engine to sleep.
|
||||
pub async fn sleep(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Query(params): Query<SleepParams>,
|
||||
params: Result<Query<SleepParams>, QueryRejection>,
|
||||
) -> Result<StatusCode, ApiError> {
|
||||
let Query(params) = params.map_err(invalid_query)?;
|
||||
|
||||
state
|
||||
.engine_core_client()
|
||||
.sleep(params.level, ¶ms.mode)
|
||||
.sleep(params.level, params.mode)
|
||||
.await
|
||||
.map_err(|error| utility_call_error("sleep", error))?;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user