[Rust Frontend]: Add /get_world_size route with static parallel size (#44801)
This commit is contained in:
@@ -414,6 +414,24 @@ impl EngineCoreClient {
|
||||
.expect("engine core client requires at least one engine")
|
||||
}
|
||||
|
||||
/// Return the world size (TP * PP) from the parallel config, if available.
|
||||
pub fn world_size(&self) -> u64 {
|
||||
self.engines
|
||||
.first()
|
||||
.expect("engine core client requires at least one engine")
|
||||
.ready_response
|
||||
.world_size
|
||||
}
|
||||
|
||||
/// Return the data parallel size from the parallel config, if available.
|
||||
pub fn data_parallel_size(&self) -> u64 {
|
||||
self.engines
|
||||
.first()
|
||||
.expect("engine core client requires at least one engine")
|
||||
.ready_response
|
||||
.data_parallel_size
|
||||
}
|
||||
|
||||
/// Get the model name associated with this client used for metrics
|
||||
/// labeling.
|
||||
pub fn model_name(&self) -> &str {
|
||||
|
||||
@@ -52,6 +52,8 @@ pub fn default_ready_response() -> EngineCoreReadyResponse {
|
||||
dp_stats_address: None,
|
||||
dtype: ModelDtype::Float32,
|
||||
vllm_version: "test-vllm-version".to_string(),
|
||||
world_size: 1,
|
||||
data_parallel_size: 1,
|
||||
kv_cache_size_tokens: None,
|
||||
kv_cache_max_concurrency: None,
|
||||
}
|
||||
|
||||
@@ -44,6 +44,10 @@ pub struct EngineCoreReadyResponse {
|
||||
pub dtype: ModelDtype,
|
||||
/// Python vLLM version reported by the engine process.
|
||||
pub vllm_version: String,
|
||||
/// World size (TP * PP) from the parallel config.
|
||||
pub world_size: u64,
|
||||
/// Data parallelism size from the parallel config.
|
||||
pub data_parallel_size: u64,
|
||||
/// Total KV cache capacity in tokens, if reported.
|
||||
pub kv_cache_size_tokens: Option<u64>,
|
||||
/// Maximum achievable request concurrency given the KV cache, if reported.
|
||||
|
||||
@@ -12,7 +12,7 @@ use crate::mock_engine::{
|
||||
MockEngineConfig, MockEngineDataSockets, connect_to_bootstrapped_frontend, connect_to_frontend,
|
||||
default_ready_response,
|
||||
};
|
||||
use crate::protocol::handshake::HandshakeInitMessage;
|
||||
use crate::protocol::handshake::{EngineCoreReadyResponse, HandshakeInitMessage};
|
||||
|
||||
/// Per-test IPC endpoint namespace backed by a unique temporary directory.
|
||||
///
|
||||
@@ -62,6 +62,15 @@ fn test_mock_engine_config() -> MockEngineConfig {
|
||||
}
|
||||
}
|
||||
|
||||
fn test_mock_engine_config_with_ready(ready_response: EngineCoreReadyResponse) -> MockEngineConfig {
|
||||
MockEngineConfig {
|
||||
local: true,
|
||||
headless: true,
|
||||
ready_response,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Complete the engine-core handshake and connect mock input/output sockets
|
||||
/// plus optional coordinator sockets.
|
||||
pub async fn setup_mock_engine_sockets(
|
||||
@@ -147,3 +156,49 @@ where
|
||||
});
|
||||
(shutdown_tx, engine_task)
|
||||
}
|
||||
|
||||
/// Like [`setup_mock_engine`] but uses a custom ready response for the
|
||||
/// handshake, allowing tests to control `world_size`, `data_parallel_size`,
|
||||
/// etc.
|
||||
async fn setup_mock_engine_with_ready(
|
||||
engine_handshake: String,
|
||||
engine_id: impl Into<EngineId>,
|
||||
ready_response: EngineCoreReadyResponse,
|
||||
) -> (DealerSocket, PushSocket) {
|
||||
let config = test_mock_engine_config_with_ready(ready_response);
|
||||
let MockEngineSockets { data_sockets, .. } =
|
||||
connect_to_frontend(engine_handshake, engine_id, config)
|
||||
.await
|
||||
.expect("connect mock engine with custom ready response");
|
||||
let MockEngineDataSockets { dealer, push } =
|
||||
data_sockets.into_iter().next().expect("mock engine data socket");
|
||||
(dealer, push)
|
||||
}
|
||||
|
||||
/// Like [`spawn_mock_engine_task`] but uses a custom ready response for the
|
||||
/// handshake, allowing tests to set `world_size` and `data_parallel_size` to
|
||||
/// non-default values.
|
||||
pub fn spawn_mock_engine_task_with_ready<F>(
|
||||
engine_handshake: String,
|
||||
engine_id: impl Into<EngineId>,
|
||||
ready_response: EngineCoreReadyResponse,
|
||||
run: F,
|
||||
) -> (oneshot::Sender<()>, tokio::task::JoinHandle<()>)
|
||||
where
|
||||
F: for<'a> FnOnce(
|
||||
&'a mut DealerSocket,
|
||||
&'a mut PushSocket,
|
||||
) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>>
|
||||
+ Send
|
||||
+ 'static,
|
||||
{
|
||||
let (shutdown_tx, shutdown_rx) = oneshot::channel();
|
||||
let engine_id = engine_id.into();
|
||||
let engine_task = tokio::spawn(async move {
|
||||
let (mut dealer, mut push) =
|
||||
setup_mock_engine_with_ready(engine_handshake, engine_id, ready_response).await;
|
||||
run(&mut dealer, &mut push).await;
|
||||
let _ = shutdown_rx.await;
|
||||
});
|
||||
(shutdown_tx, engine_task)
|
||||
}
|
||||
|
||||
@@ -358,6 +358,8 @@ class EngineCoreReadyResponse:
|
||||
dp_stats_address: str | None
|
||||
dtype: str
|
||||
vllm_version: str
|
||||
world_size: int
|
||||
data_parallel_size: int
|
||||
kv_cache_size_tokens: int | None = None
|
||||
kv_cache_max_concurrency: float | None = None
|
||||
|
||||
@@ -369,6 +371,8 @@ ready_response = EngineCoreReadyResponse(
|
||||
dp_stats_address=None,
|
||||
dtype="float32",
|
||||
vllm_version="0.0.0",
|
||||
data_parallel_size=1,
|
||||
world_size=1,
|
||||
)
|
||||
|
||||
print(msgspec.msgpack.encode(request).hex())
|
||||
|
||||
Reference in New Issue
Block a user