[Rust Frontend]: Add /get_world_size route with static parallel size (#44801)
This commit is contained in:
@@ -414,6 +414,24 @@ impl EngineCoreClient {
|
||||
.expect("engine core client requires at least one engine")
|
||||
}
|
||||
|
||||
/// Return the world size (TP * PP) from the parallel config, if available.
|
||||
pub fn world_size(&self) -> u64 {
|
||||
self.engines
|
||||
.first()
|
||||
.expect("engine core client requires at least one engine")
|
||||
.ready_response
|
||||
.world_size
|
||||
}
|
||||
|
||||
/// Return the data parallel size from the parallel config, if available.
|
||||
pub fn data_parallel_size(&self) -> u64 {
|
||||
self.engines
|
||||
.first()
|
||||
.expect("engine core client requires at least one engine")
|
||||
.ready_response
|
||||
.data_parallel_size
|
||||
}
|
||||
|
||||
/// Get the model name associated with this client used for metrics
|
||||
/// labeling.
|
||||
pub fn model_name(&self) -> &str {
|
||||
|
||||
@@ -52,6 +52,8 @@ pub fn default_ready_response() -> EngineCoreReadyResponse {
|
||||
dp_stats_address: None,
|
||||
dtype: ModelDtype::Float32,
|
||||
vllm_version: "test-vllm-version".to_string(),
|
||||
world_size: 1,
|
||||
data_parallel_size: 1,
|
||||
kv_cache_size_tokens: None,
|
||||
kv_cache_max_concurrency: None,
|
||||
}
|
||||
|
||||
@@ -44,6 +44,10 @@ pub struct EngineCoreReadyResponse {
|
||||
pub dtype: ModelDtype,
|
||||
/// Python vLLM version reported by the engine process.
|
||||
pub vllm_version: String,
|
||||
/// World size (TP * PP) from the parallel config.
|
||||
pub world_size: u64,
|
||||
/// Data parallelism size from the parallel config.
|
||||
pub data_parallel_size: u64,
|
||||
/// Total KV cache capacity in tokens, if reported.
|
||||
pub kv_cache_size_tokens: Option<u64>,
|
||||
/// Maximum achievable request concurrency given the KV cache, if reported.
|
||||
|
||||
@@ -12,7 +12,7 @@ use crate::mock_engine::{
|
||||
MockEngineConfig, MockEngineDataSockets, connect_to_bootstrapped_frontend, connect_to_frontend,
|
||||
default_ready_response,
|
||||
};
|
||||
use crate::protocol::handshake::HandshakeInitMessage;
|
||||
use crate::protocol::handshake::{EngineCoreReadyResponse, HandshakeInitMessage};
|
||||
|
||||
/// Per-test IPC endpoint namespace backed by a unique temporary directory.
|
||||
///
|
||||
@@ -62,6 +62,15 @@ fn test_mock_engine_config() -> MockEngineConfig {
|
||||
}
|
||||
}
|
||||
|
||||
fn test_mock_engine_config_with_ready(ready_response: EngineCoreReadyResponse) -> MockEngineConfig {
|
||||
MockEngineConfig {
|
||||
local: true,
|
||||
headless: true,
|
||||
ready_response,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Complete the engine-core handshake and connect mock input/output sockets
|
||||
/// plus optional coordinator sockets.
|
||||
pub async fn setup_mock_engine_sockets(
|
||||
@@ -147,3 +156,49 @@ where
|
||||
});
|
||||
(shutdown_tx, engine_task)
|
||||
}
|
||||
|
||||
/// Like [`setup_mock_engine`] but uses a custom ready response for the
|
||||
/// handshake, allowing tests to control `world_size`, `data_parallel_size`,
|
||||
/// etc.
|
||||
async fn setup_mock_engine_with_ready(
|
||||
engine_handshake: String,
|
||||
engine_id: impl Into<EngineId>,
|
||||
ready_response: EngineCoreReadyResponse,
|
||||
) -> (DealerSocket, PushSocket) {
|
||||
let config = test_mock_engine_config_with_ready(ready_response);
|
||||
let MockEngineSockets { data_sockets, .. } =
|
||||
connect_to_frontend(engine_handshake, engine_id, config)
|
||||
.await
|
||||
.expect("connect mock engine with custom ready response");
|
||||
let MockEngineDataSockets { dealer, push } =
|
||||
data_sockets.into_iter().next().expect("mock engine data socket");
|
||||
(dealer, push)
|
||||
}
|
||||
|
||||
/// Like [`spawn_mock_engine_task`] but uses a custom ready response for the
|
||||
/// handshake, allowing tests to set `world_size` and `data_parallel_size` to
|
||||
/// non-default values.
|
||||
pub fn spawn_mock_engine_task_with_ready<F>(
|
||||
engine_handshake: String,
|
||||
engine_id: impl Into<EngineId>,
|
||||
ready_response: EngineCoreReadyResponse,
|
||||
run: F,
|
||||
) -> (oneshot::Sender<()>, tokio::task::JoinHandle<()>)
|
||||
where
|
||||
F: for<'a> FnOnce(
|
||||
&'a mut DealerSocket,
|
||||
&'a mut PushSocket,
|
||||
) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>>
|
||||
+ Send
|
||||
+ 'static,
|
||||
{
|
||||
let (shutdown_tx, shutdown_rx) = oneshot::channel();
|
||||
let engine_id = engine_id.into();
|
||||
let engine_task = tokio::spawn(async move {
|
||||
let (mut dealer, mut push) =
|
||||
setup_mock_engine_with_ready(engine_handshake, engine_id, ready_response).await;
|
||||
run(&mut dealer, &mut push).await;
|
||||
let _ = shutdown_rx.await;
|
||||
});
|
||||
(shutdown_tx, engine_task)
|
||||
}
|
||||
|
||||
@@ -358,6 +358,8 @@ class EngineCoreReadyResponse:
|
||||
dp_stats_address: str | None
|
||||
dtype: str
|
||||
vllm_version: str
|
||||
world_size: int
|
||||
data_parallel_size: int
|
||||
kv_cache_size_tokens: int | None = None
|
||||
kv_cache_max_concurrency: float | None = None
|
||||
|
||||
@@ -369,6 +371,8 @@ ready_response = EngineCoreReadyResponse(
|
||||
dp_stats_address=None,
|
||||
dtype="float32",
|
||||
vllm_version="0.0.0",
|
||||
data_parallel_size=1,
|
||||
world_size=1,
|
||||
)
|
||||
|
||||
print(msgspec.msgpack.encode(request).hex())
|
||||
|
||||
@@ -12,6 +12,7 @@ mod server_info;
|
||||
mod sleep;
|
||||
mod tokenize;
|
||||
mod version;
|
||||
mod world_size;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -100,6 +101,7 @@ fn build_router_with_options(
|
||||
.route("/resume", post(pause::resume))
|
||||
.route("/is_paused", get(pause::is_paused))
|
||||
.route("/server_info", get(server_info::server_info))
|
||||
.route("/get_world_size", get(world_size::get_world_size))
|
||||
}
|
||||
|
||||
let enable_request_id_headers = state.api_server_options.enable_request_id_headers;
|
||||
|
||||
@@ -23,6 +23,7 @@ use vllm_chat::{
|
||||
ChatTextBackend, DefaultChatOutputProcessor, DynChatOutputProcessor, DynChatRenderer,
|
||||
NewChatOutputProcessorOptions,
|
||||
};
|
||||
use vllm_engine_core_client::mock_engine::default_ready_response;
|
||||
use vllm_engine_core_client::protocol::logprobs::{
|
||||
Logprobs, MaybeWireLogprobs, PositionLogprobs, TokenLogprob,
|
||||
};
|
||||
@@ -31,7 +32,9 @@ use vllm_engine_core_client::protocol::{
|
||||
EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, EngineCoreRequest, StopReason,
|
||||
decode_value,
|
||||
};
|
||||
use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
|
||||
use vllm_engine_core_client::test_utils::{
|
||||
IpcNamespace, spawn_mock_engine_task, spawn_mock_engine_task_with_ready,
|
||||
};
|
||||
use vllm_engine_core_client::{
|
||||
ENGINE_CORE_DEAD_SENTINEL, EngineCoreClient, EngineCoreClientConfig, EngineId,
|
||||
};
|
||||
@@ -788,6 +791,45 @@ async fn test_app_with_dev_mode(dev_mode_enabled: bool) -> axum::Router {
|
||||
)
|
||||
}
|
||||
|
||||
/// Build a dev-mode router backed by a mock engine using a custom ready
|
||||
/// response, returning the router and the engine task handle so the engine
|
||||
/// stays alive for the duration of the test.
|
||||
async fn test_dev_mode_app_with_ready(
|
||||
ready_response: vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse,
|
||||
) -> (axum::Router, MockEngineTask) {
|
||||
let ipc = IpcNamespace::new().expect("create ipc namespace");
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
let engine_id = b"engine-world-size".to_vec();
|
||||
|
||||
let engine_task = MockEngineTask::new(spawn_mock_engine_task_with_ready(
|
||||
handshake_address.clone(),
|
||||
engine_id.clone(),
|
||||
ready_response,
|
||||
|_dealer, _push| boxed_test_future(async {}),
|
||||
));
|
||||
|
||||
let client = EngineCoreClient::connect(
|
||||
EngineCoreClientConfig::new_single(handshake_address)
|
||||
.with_model_name("test-model")
|
||||
.with_local_input_output_addresses(
|
||||
Some(ipc.input_endpoint()),
|
||||
Some(ipc.output_endpoint()),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("connect client");
|
||||
|
||||
let chat = ChatLlm::from_shared_backend(test_llm(client), Arc::new(FakeChatBackend::new()));
|
||||
let app = build_router_with_dev_mode(
|
||||
Arc::new(AppState::new(
|
||||
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
|
||||
chat,
|
||||
)),
|
||||
true,
|
||||
);
|
||||
(app, engine_task)
|
||||
}
|
||||
|
||||
async fn test_app_with_request_id_headers() -> (axum::Router, MockEngineTask) {
|
||||
let (chat, engine_task) = test_models_with_engine_outputs_and_backend(
|
||||
b"engine-openai-request-id",
|
||||
@@ -5876,3 +5918,72 @@ async fn tokenize_chat_continue_final_vs_new_assistant_differs() {
|
||||
let new_len = new_assistant["tokens"].as_array().unwrap().len();
|
||||
assert!(new_len > continue_len);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn world_size_endpoint_is_dev_mode_only() {
|
||||
let mut app = test_app().await;
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.uri("/get_world_size")
|
||||
.body(Body::empty())
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn world_size_includes_data_parallelism_by_default() {
|
||||
let ready = vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse {
|
||||
world_size: 2,
|
||||
data_parallel_size: 4,
|
||||
..default_ready_response()
|
||||
};
|
||||
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready).await;
|
||||
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.uri("/get_world_size")
|
||||
.body(Body::empty())
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
|
||||
assert_eq!(json, json!({"world_size": 8}));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn world_size_excludes_data_parallelism_when_include_dp_false() {
|
||||
let ready = vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse {
|
||||
world_size: 2,
|
||||
data_parallel_size: 4,
|
||||
..default_ready_response()
|
||||
};
|
||||
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready).await;
|
||||
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.uri("/get_world_size?include_dp=false")
|
||||
.body(Body::empty())
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
|
||||
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
|
||||
assert_eq!(json, json!({"world_size": 2}));
|
||||
}
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::Json;
|
||||
use axum::extract::{Query, State};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::error::ApiError;
|
||||
use crate::state::AppState;
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub(crate) struct WorldSizeParams {
|
||||
/// If true (default), returns the world size including data parallelism
|
||||
/// (TP * PP * DP). If false, returns the world size without data
|
||||
/// parallelism (TP * PP).
|
||||
#[serde(default = "default_true")]
|
||||
include_dp: bool,
|
||||
}
|
||||
|
||||
const fn default_true() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub(crate) struct WorldSizeResponse {
|
||||
world_size: u64,
|
||||
}
|
||||
|
||||
/// Get the world size from the parallel config.
|
||||
///
|
||||
/// Currently reads static values captured during the engine startup handshake.
|
||||
///
|
||||
/// TODO: If the world size can change at runtime (e.g. elastic EP scaling,
|
||||
/// DP rank recovery), this should be switched to either:
|
||||
/// - A `call_utility("get_world_size", (include_dp,))` RPC to the Python
|
||||
/// engine for live values (simple, adds one ZMQ round-trip per request), or
|
||||
/// - A push-based approach where the engine sends config updates via the
|
||||
/// output stream into shared state (zero per-request overhead, more complex).
|
||||
pub async fn get_world_size(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Query(params): Query<WorldSizeParams>,
|
||||
) -> Result<Json<WorldSizeResponse>, ApiError> {
|
||||
let client = state.engine_core_client();
|
||||
|
||||
let ws = client.world_size();
|
||||
|
||||
let world_size = if params.include_dp {
|
||||
let dp = client.data_parallel_size();
|
||||
ws * dp
|
||||
} else {
|
||||
ws
|
||||
};
|
||||
|
||||
Ok(Json(WorldSizeResponse { world_size }))
|
||||
}
|
||||
Reference in New Issue
Block a user