[Rust Frontend] Add /version endpoint using engine-reported value (#43854)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-05-29 00:32:27 +00:00
committed by GitHub
parent b690b2bb67
commit 1521173c17
17 changed files with 159 additions and 195 deletions
+23 -21
View File
@@ -290,10 +290,8 @@ impl EngineCoreClient {
// If any engine reported a dp_stats_address in its ready response, use it
// as the external coordinator address.
let dp_stats_address: Option<String> = engines
.iter()
.filter_map(|e| e.ready_response.as_ref())
.find_map(|r| r.dp_stats_address.clone());
let dp_stats_address: Option<String> =
engines.iter().find_map(|engine| engine.ready_response.dp_stats_address.clone());
let (coordinator, coordinator_output_task, coordinator_task) =
if let Some(coordinator_transport) = connected.coordinator {
@@ -368,40 +366,44 @@ impl EngineCoreClient {
/// Return the ready responses received from all engines on the input
/// socket.
pub fn ready_responses(&self) -> Vec<&EngineCoreReadyResponse> {
self.engines
.iter()
.filter_map(|engine| engine.ready_response.as_ref())
.collect()
self.engines.iter().map(|engine| &engine.ready_response).collect()
}
/// Return the engine-reported effective model dtype, when available.
pub fn model_dtype(&self) -> Option<ModelDtype> {
/// Return the engine-reported effective model dtype.
pub fn model_dtype(&self) -> ModelDtype {
self.engines
.iter()
.filter_map(|engine| engine.ready_response.as_ref())
.find_map(|response| response.dtype)
.first()
.expect("engine core client requires at least one engine")
.ready_response
.dtype
}
/// Return the engine-reported Python vLLM version.
pub fn vllm_version(&self) -> &str {
self.engines
.first()
.expect("engine core client requires at least one engine")
.ready_response
.vllm_version
.as_str()
}
/// Return the total number of GPU blocks summed across all connected
/// engines.
pub fn total_num_gpu_blocks(&self) -> u64 {
self.engines
.iter()
.filter_map(|engine| engine.ready_response.as_ref())
.map(|r| r.num_gpu_blocks)
.sum()
self.engines.iter().map(|engine| engine.ready_response.num_gpu_blocks).sum()
}
/// Return the minimum engine-reported `max_model_len` across all engines.
///
/// This is the auto-fitted value after KV cache profiling and may differ
/// from the originally configured value.
pub fn max_model_len(&self) -> Option<u32> {
pub fn max_model_len(&self) -> u32 {
self.engines
.iter()
.filter_map(|e| e.ready_response.as_ref())
.map(|r| r.max_model_len as u32)
.map(|engine| engine.ready_response.max_model_len as u32)
.min()
.expect("engine core client requires at least one engine")
}
/// Get the model name associated with this client used for metrics
@@ -382,6 +382,7 @@ mod tests {
use zeromq::{RouterSocket, Socket};
use super::*;
use crate::mock_engine::default_ready_response;
async fn test_inner() -> ClientInner {
let mut socket = RouterSocket::new();
@@ -392,7 +393,7 @@ mod tests {
"test-model".to_string(),
&[ConnectedEngine {
engine_id: EngineId::from(b"engine-0"),
ready_response: None,
ready_response: default_ready_response(),
}],
)
}
+25 -68
View File
@@ -339,15 +339,20 @@ mod tests {
use super::{EngineRoutingState, RequestRegistry, UtilityRegistry};
use crate::EngineId;
use crate::client::state::EngineLoadSnapshot;
use crate::mock_engine::default_ready_response;
use crate::protocol::{EngineCoreFinishReason, EngineCoreOutput};
use crate::transport::ConnectedEngine;
fn connected_engine(engine_id: EngineId) -> ConnectedEngine {
ConnectedEngine {
engine_id,
ready_response: default_ready_response(),
}
}
#[test]
fn registry_rejects_duplicate_request_ids() {
let mut registry = RequestRegistry::new(&[ConnectedEngine {
engine_id: EngineId::from(b"engine-0"),
ready_response: None,
}]);
let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
registry.register("req-1".to_string(), None).unwrap();
let error = registry.register("req-1".to_string(), None).unwrap_err();
assert!(matches!(
@@ -358,10 +363,7 @@ mod tests {
#[test]
fn registry_removes_finished_request_on_output() {
let mut registry = RequestRegistry::new(&[ConnectedEngine {
engine_id: EngineId::from(b"engine-0"),
ready_response: None,
}]);
let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
registry.register("req-1".to_string(), None).unwrap();
let sender = registry.sender_for_output(&EngineCoreOutput {
@@ -376,10 +378,7 @@ mod tests {
#[test]
fn registry_closes_all_requests_on_failure() {
let mut registry = RequestRegistry::new(&[ConnectedEngine {
engine_id: EngineId::from(b"engine-0"),
ready_response: None,
}]);
let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
registry.register("req-1".to_string(), None).unwrap();
registry.register("req-2".to_string(), None).unwrap();
@@ -394,14 +393,8 @@ mod tests {
let engine_0 = EngineId::from_engine_index(0);
let engine_1 = EngineId::from_engine_index(1);
let mut registry = RequestRegistry::new(&[
ConnectedEngine {
engine_id: engine_0.clone(),
ready_response: None,
},
ConnectedEngine {
engine_id: engine_1.clone(),
ready_response: None,
},
connected_engine(engine_0.clone()),
connected_engine(engine_1.clone()),
]);
let (chosen_0, _) = registry.register("req-1".to_string(), None).unwrap();
let (chosen_1, _) = registry.register("req-2".to_string(), None).unwrap();
@@ -428,14 +421,8 @@ mod tests {
let engine_0 = EngineId::from_engine_index(0);
let engine_1 = EngineId::from_engine_index(1);
let mut registry = RequestRegistry::new(&[
ConnectedEngine {
engine_id: engine_0.clone(),
ready_response: None,
},
ConnectedEngine {
engine_id: engine_1.clone(),
ready_response: None,
},
connected_engine(engine_0.clone()),
connected_engine(engine_1.clone()),
]);
let (chosen_0, _) = registry.register("req-1".to_string(), None).unwrap();
@@ -488,14 +475,8 @@ mod tests {
let engine_0 = EngineId::from_engine_index(0);
let engine_1 = EngineId::from_engine_index(1);
let mut registry = RequestRegistry::new(&[
ConnectedEngine {
engine_id: engine_0.clone(),
ready_response: None,
},
ConnectedEngine {
engine_id: engine_1.clone(),
ready_response: None,
},
connected_engine(engine_0.clone()),
connected_engine(engine_1.clone()),
]);
assert!(registry.apply_scheduler_counts(
@@ -523,18 +504,9 @@ mod tests {
let engine_1 = EngineId::from_engine_index(1);
let engine_2 = EngineId::from_engine_index(2);
let mut registry = RequestRegistry::new(&[
ConnectedEngine {
engine_id: engine_0.clone(),
ready_response: None,
},
ConnectedEngine {
engine_id: engine_1.clone(),
ready_response: None,
},
ConnectedEngine {
engine_id: engine_2.clone(),
ready_response: None,
},
connected_engine(engine_0.clone()),
connected_engine(engine_1.clone()),
connected_engine(engine_2.clone()),
]);
// Explicitly target rank 2 (third engine).
@@ -555,14 +527,8 @@ mod tests {
let engine_0 = EngineId::from_engine_index(0);
let engine_1 = EngineId::from_engine_index(1);
let mut registry = RequestRegistry::new(&[
ConnectedEngine {
engine_id: engine_0.clone(),
ready_response: None,
},
ConnectedEngine {
engine_id: engine_1.clone(),
ready_response: None,
},
connected_engine(engine_0.clone()),
connected_engine(engine_1.clone()),
]);
// Load-balance: first two go to engine_0 and engine_1.
@@ -577,14 +543,8 @@ mod tests {
#[test]
fn register_with_out_of_range_rank_returns_error() {
let mut registry = RequestRegistry::new(&[
ConnectedEngine {
engine_id: EngineId::from_engine_index(0),
ready_response: None,
},
ConnectedEngine {
engine_id: EngineId::from_engine_index(1),
ready_response: None,
},
connected_engine(EngineId::from_engine_index(0)),
connected_engine(EngineId::from_engine_index(1)),
]);
let error = registry.register("req-1".to_string(), Some(2)).unwrap_err();
@@ -600,10 +560,7 @@ mod tests {
#[test]
fn register_with_rank_on_single_engine_only_accepts_zero() {
let engine_0 = EngineId::from_engine_index(0);
let mut registry = RequestRegistry::new(&[ConnectedEngine {
engine_id: engine_0.clone(),
ready_response: None,
}]);
let mut registry = RequestRegistry::new(&[connected_engine(engine_0.clone())]);
let (chosen, _) = registry.register("req-ok".to_string(), Some(0)).unwrap();
assert_eq!(chosen, engine_0);
@@ -47,7 +47,8 @@ pub fn default_ready_response() -> EngineCoreReadyResponse {
max_model_len: DEFAULT_MOCK_MAX_MODEL_LEN,
num_gpu_blocks: DEFAULT_MOCK_NUM_GPU_BLOCKS,
dp_stats_address: None,
dtype: Some(ModelDtype::Float32),
dtype: ModelDtype::Float32,
vllm_version: "test-vllm-version".to_string(),
}
}
@@ -39,10 +39,9 @@ pub struct EngineCoreReadyResponse {
/// DP coordinator stats publish address, if applicable.
pub dp_stats_address: Option<String>,
/// Effective model dtype after Python vLLM resolves `--dtype`.
// TODO: This is currently not wired up on the engine side. After it's added, remove `Option`
// and `serde(default)`.
#[serde(default)]
pub dtype: Option<ModelDtype>,
pub dtype: ModelDtype,
/// Python vLLM version reported by the engine process.
pub vllm_version: String,
}
/// Frontend-owned ZMQ addresses that are sent to the engine during startup
@@ -69,22 +68,3 @@ pub struct HandshakeInitMessage {
pub addresses: HandshakeAddresses,
pub parallel_config: BTreeMap<String, OpaqueValue>,
}
#[cfg(test)]
mod tests {
use super::EngineCoreReadyResponse;
use crate::protocol::ModelDtype;
#[test]
fn ready_response_accepts_effective_dtype() {
let response: EngineCoreReadyResponse = serde_json::from_value(serde_json::json!({
"max_model_len": 4096,
"num_gpu_blocks": 2,
"dp_stats_address": null,
"dtype": "bfloat16"
}))
.unwrap();
assert_eq!(response.dtype, Some(ModelDtype::BFloat16));
}
}
@@ -10,9 +10,9 @@ use crate::EngineId;
pub use crate::mock_engine::{MockCoordinatorSockets, MockEngineSockets};
use crate::mock_engine::{
MockEngineConfig, MockEngineDataSockets, connect_to_bootstrapped_frontend, connect_to_frontend,
default_ready_response,
};
use crate::protocol::ModelDtype;
use crate::protocol::handshake::{EngineCoreReadyResponse, HandshakeInitMessage};
use crate::protocol::handshake::HandshakeInitMessage;
/// Per-test IPC endpoint namespace backed by a unique temporary directory.
///
@@ -57,12 +57,7 @@ fn test_mock_engine_config() -> MockEngineConfig {
MockEngineConfig {
local: true,
headless: true,
ready_response: EngineCoreReadyResponse {
max_model_len: 4096,
num_gpu_blocks: 0,
dp_stats_address: None,
dtype: Some(ModelDtype::Float32),
},
ready_response: default_ready_response(),
..Default::default()
}
}
@@ -925,6 +925,7 @@ async fn client_fail_closes_when_main_output_path_receives_dp_control() {
.await;
assert_eq!(client.engine_identities()[0], b"engine-0");
assert!(client.ready_responses()[0].max_model_len > 0);
assert_eq!(client.vllm_version(), "test-vllm-version");
let mut stream_1 = client.call(sample_request_with_id("req-1")).await.unwrap();
let mut stream_2 = client.call(sample_request_with_id("req-2")).await.unwrap();
+40 -50
View File
@@ -104,8 +104,8 @@ pub struct ConnectedEngine {
/// The identity of the connected engine.
pub engine_id: EngineId,
/// Post-initialization configuration received from the engine on the input
/// socket registration message. `None` until the registration is received.
pub ready_response: Option<EngineCoreReadyResponse>,
/// socket registration message.
pub ready_response: EngineCoreReadyResponse,
}
/// Represents the connected shared transport plus all registered engines after
@@ -295,18 +295,9 @@ pub async fn connect_handshake(
}
}
// 4. Wait for every engine to connect to the shared input socket and register itself. The
// `ready_response` is a placeholder; it is populated for each engine by
// `wait_for_input_registrations` below.
let mut engines: Vec<_> = engines
.into_keys()
.map(|engine_id| ConnectedEngine {
engine_id,
ready_response: None,
})
.collect();
wait_for_input_registrations(&mut input_socket, &mut engines, ready_timeout).await?;
// 6. Wait for every engine to connect to the shared input socket and register itself.
let engines =
wait_for_input_registrations(&mut input_socket, engines.into_keys(), ready_timeout).await?;
debug!(
engine_count = engines.len(),
"all engines registered on shared input socket"
@@ -349,15 +340,13 @@ pub async fn connect_bootstrapped(
let mut output_socket = PullSocket::new();
let output_address = output_socket.bind(output_address).await?.to_string();
// TODO: follow start rank
let mut engines = (0..engine_count)
.map(|index| ConnectedEngine {
engine_id: EngineId::from((index as u16).to_le_bytes().to_vec()),
ready_response: None,
})
.collect::<Vec<_>>();
wait_for_input_registrations(&mut input_socket, &mut engines, ready_timeout).await?;
let engines = wait_for_input_registrations(
&mut input_socket,
// TODO: follow start rank
(0..engine_count).map(|index| EngineId::from((index as u16).to_le_bytes().to_vec())),
ready_timeout,
)
.await?;
info!(
engine_count = engines.len(),
"bootstrapped engines connected"
@@ -455,17 +444,14 @@ async fn send_init_message(
/// Simplify API server handshake"), the payload is a msgpack-encoded
/// [`EngineCoreReadyResponse`] carrying post-initialization values such as
/// `max_model_len`.
///
/// Older engines sent an empty second frame here just to establish the
/// ROUTER/DEALER backchannel, with no structured payload on the input socket.
/// We continue to tolerate that legacy shape so the frontend can still connect
/// to slightly older local engine checkouts.
async fn wait_for_input_registrations(
input_socket: &mut RouterSocket,
engines: &mut [ConnectedEngine],
expected_engines: impl IntoIterator<Item = EngineId>,
ready_timeout: Duration,
) -> Result<()> {
let mut pending = engines.iter().map(|e| e.engine_id.clone()).collect::<BTreeSet<_>>();
) -> Result<Vec<ConnectedEngine>> {
let expected_engines = expected_engines.into_iter().collect::<Vec<_>>();
let mut pending = expected_engines.iter().cloned().collect::<BTreeSet<_>>();
let mut ready_responses = BTreeMap::new();
while !pending.is_empty() {
let registration = timeout(ready_timeout, input_socket.recv()).await.map_err(|_| {
@@ -489,29 +475,33 @@ async fn wait_for_input_registrations(
);
}
let ready_response = if frames[1].is_empty() {
debug!(
?actual_id,
"received legacy empty input registration from engine"
if frames[1].is_empty() {
bail_unexpected_handshake_message!(
"expected msgpack EngineCoreReadyResponse for engine input registration, got empty payload from engine id {actual_id:?}"
);
None
} else {
let ready_response: EngineCoreReadyResponse = decode_msgpack(&frames[1])?;
debug!(
?actual_id,
?ready_response,
"received input registration from engine"
);
Some(ready_response)
};
// Store the ready response in the corresponding engine entry.
if let Some(engine) = engines.iter_mut().find(|e| e.engine_id == actual_id) {
engine.ready_response = ready_response;
}
let ready_response: EngineCoreReadyResponse = decode_msgpack(&frames[1])?;
debug!(
?actual_id,
?ready_response,
"received input registration from engine"
);
ready_responses.insert(actual_id, ready_response);
}
Ok(())
Ok(expected_engines
.into_iter()
.map(|engine_id| {
let ready_response = ready_responses
.remove(&engine_id)
.expect("every expected engine id has a decoded ready response");
ConnectedEngine {
engine_id,
ready_response,
}
})
.collect())
}
/// Send an encoded message to the engine through the input socket.