[Rust Frontend]: Add /get_world_size route with static parallel size (#44801)

This commit is contained in:
Ashar
2026-06-18 13:10:20 +00:00
committed by GitHub
parent 22cc891108
commit 4cb5e746b6
11 changed files with 258 additions and 2 deletions
+2
View File
@@ -12,6 +12,7 @@ mod server_info;
mod sleep;
mod tokenize;
mod version;
mod world_size;
use std::sync::Arc;
@@ -100,6 +101,7 @@ fn build_router_with_options(
.route("/resume", post(pause::resume))
.route("/is_paused", get(pause::is_paused))
.route("/server_info", get(server_info::server_info))
.route("/get_world_size", get(world_size::get_world_size))
}
let enable_request_id_headers = state.api_server_options.enable_request_id_headers;
+112 -1
View File
@@ -23,6 +23,7 @@ use vllm_chat::{
ChatTextBackend, DefaultChatOutputProcessor, DynChatOutputProcessor, DynChatRenderer,
NewChatOutputProcessorOptions,
};
use vllm_engine_core_client::mock_engine::default_ready_response;
use vllm_engine_core_client::protocol::logprobs::{
Logprobs, MaybeWireLogprobs, PositionLogprobs, TokenLogprob,
};
@@ -31,7 +32,9 @@ use vllm_engine_core_client::protocol::{
EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, EngineCoreRequest, StopReason,
decode_value,
};
use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
use vllm_engine_core_client::test_utils::{
IpcNamespace, spawn_mock_engine_task, spawn_mock_engine_task_with_ready,
};
use vllm_engine_core_client::{
ENGINE_CORE_DEAD_SENTINEL, EngineCoreClient, EngineCoreClientConfig, EngineId,
};
@@ -788,6 +791,45 @@ async fn test_app_with_dev_mode(dev_mode_enabled: bool) -> axum::Router {
)
}
/// Build a dev-mode router backed by a mock engine using a custom ready
/// response, returning the router and the engine task handle so the engine
/// stays alive for the duration of the test.
async fn test_dev_mode_app_with_ready(
ready_response: vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse,
) -> (axum::Router, MockEngineTask) {
let ipc = IpcNamespace::new().expect("create ipc namespace");
let handshake_address = ipc.handshake_endpoint();
let engine_id = b"engine-world-size".to_vec();
let engine_task = MockEngineTask::new(spawn_mock_engine_task_with_ready(
handshake_address.clone(),
engine_id.clone(),
ready_response,
|_dealer, _push| boxed_test_future(async {}),
));
let client = EngineCoreClient::connect(
EngineCoreClientConfig::new_single(handshake_address)
.with_model_name("test-model")
.with_local_input_output_addresses(
Some(ipc.input_endpoint()),
Some(ipc.output_endpoint()),
),
)
.await
.expect("connect client");
let chat = ChatLlm::from_shared_backend(test_llm(client), Arc::new(FakeChatBackend::new()));
let app = build_router_with_dev_mode(
Arc::new(AppState::new(
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
chat,
)),
true,
);
(app, engine_task)
}
async fn test_app_with_request_id_headers() -> (axum::Router, MockEngineTask) {
let (chat, engine_task) = test_models_with_engine_outputs_and_backend(
b"engine-openai-request-id",
@@ -5876,3 +5918,72 @@ async fn tokenize_chat_continue_final_vs_new_assistant_differs() {
let new_len = new_assistant["tokens"].as_array().unwrap().len();
assert!(new_len > continue_len);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn world_size_endpoint_is_dev_mode_only() {
let mut app = test_app().await;
let response = app
.call(
Request::builder()
.uri("/get_world_size")
.body(Body::empty())
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::NOT_FOUND);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn world_size_includes_data_parallelism_by_default() {
let ready = vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse {
world_size: 2,
data_parallel_size: 4,
..default_ready_response()
};
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready).await;
let response = app
.call(
Request::builder()
.uri("/get_world_size")
.body(Body::empty())
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::OK);
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
assert_eq!(json, json!({"world_size": 8}));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn world_size_excludes_data_parallelism_when_include_dp_false() {
let ready = vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse {
world_size: 2,
data_parallel_size: 4,
..default_ready_response()
};
let (mut app, _engine_task) = test_dev_mode_app_with_ready(ready).await;
let response = app
.call(
Request::builder()
.uri("/get_world_size?include_dp=false")
.body(Body::empty())
.expect("build request"),
)
.await
.expect("call app");
assert_eq!(response.status(), StatusCode::OK);
let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
assert_eq!(json, json!({"world_size": 2}));
}
+54
View File
@@ -0,0 +1,54 @@
use std::sync::Arc;
use axum::Json;
use axum::extract::{Query, State};
use serde::{Deserialize, Serialize};
use crate::error::ApiError;
use crate::state::AppState;
#[derive(Debug, Deserialize)]
pub(crate) struct WorldSizeParams {
/// If true (default), returns the world size including data parallelism
/// (TP * PP * DP). If false, returns the world size without data
/// parallelism (TP * PP).
#[serde(default = "default_true")]
include_dp: bool,
}
const fn default_true() -> bool {
true
}
#[derive(Serialize)]
pub(crate) struct WorldSizeResponse {
world_size: u64,
}
/// Get the world size from the parallel config.
///
/// Currently reads static values captured during the engine startup handshake.
///
/// TODO: If the world size can change at runtime (e.g. elastic EP scaling,
/// DP rank recovery), this should be switched to either:
/// - A `call_utility("get_world_size", (include_dp,))` RPC to the Python
/// engine for live values (simple, adds one ZMQ round-trip per request), or
/// - A push-based approach where the engine sends config updates via the
/// output stream into shared state (zero per-request overhead, more complex).
pub async fn get_world_size(
State(state): State<Arc<AppState>>,
Query(params): Query<WorldSizeParams>,
) -> Result<Json<WorldSizeResponse>, ApiError> {
let client = state.engine_core_client();
let ws = client.world_size();
let world_size = if params.include_dp {
let dp = client.data_parallel_size();
ws * dp
} else {
ws
};
Ok(Json(WorldSizeResponse { world_size }))
}