[Rust Frontend] Move code from vllm-frontend-rs (#43283)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Signed-off-by: Eric Curtin <eric.curtin@docker.com>
Signed-off-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com>
Signed-off-by: Will.hou <1205157517@qq.com>
Signed-off-by: Will.hou <willamhou@ceresman.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Eric Curtin <eric.curtin@docker.com>
Co-authored-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com>
Co-authored-by: Will.hou <1205157517@qq.com>
Co-authored-by: Will.hou <willamhou@ceresman.com>

Please see https://github.com/Inferact/vllm-frontend-rs for full original commit history.
This commit is contained in:
Bugen Zhao
2026-05-21 17:21:48 -07:00
committed by GitHub
co-authored by Nick Hill Eric Curtin Dev-X25874 Will.hou Will.hou
parent 39d5fa96a7
commit 39910f2b25
284 changed files with 72569 additions and 90 deletions
+49
View File
@@ -0,0 +1,49 @@
[package]
name = "vllm-engine-core-client"
version.workspace = true
edition.workspace = true
license.workspace = true
[features]
test-util = ["dep:tempfile"]
[dependencies]
arc-swap.workspace = true
bytemuck.workspace = true
byteorder.workspace = true
bytes.workspace = true
easy-ext.workspace = true
enum-as-inner.workspace = true
futures.workspace = true
half.workspace = true
hex.workspace = true
itertools.workspace = true
parking_lot.workspace = true
rmp-serde.workspace = true
rmpv.workspace = true
serde.workspace = true
serde_default.workspace = true
serde_json.workspace = true
serde_repr.workspace = true
serde_tuple.workspace = true
serde_with.workspace = true
task-local.workspace = true
tempfile = { workspace = true, optional = true }
thiserror.workspace = true
thiserror-ext.workspace = true
tokio.workspace = true
tokio-util.workspace = true
tracing.workspace = true
vllm-metrics.workspace = true
zeromq.workspace = true
[dev-dependencies]
anyhow.workspace = true
clap.workspace = true
expect-test.workspace = true
hex.workspace = true
tempfile.workspace = true
tracing-subscriber.workspace = true
[lints]
workspace = true
@@ -0,0 +1,53 @@
# Engine-Core Smoke Tests
Start headless `vllm`:
```bash
source ../vllm/.venv/bin/activate
HF_HUB_OFFLINE=1 \
VLLM_LOGGING_LEVEL=DEBUG \
VLLM_CPU_KVCACHE_SPACE=2 \
VLLM_HOST_IP=127.0.0.1 \
VLLM_LOOPBACK_IP=127.0.0.1 \
python3 -m vllm.entrypoints.cli.main serve Qwen/Qwen3-0.6B \
--headless \
--enable-sleep-mode \
--data-parallel-address 127.0.0.1 \
--data-parallel-rpc-port 62100 \
--data-parallel-size-local 1 \
--max-model-len 512 \
--dtype float16
```
Run the Rust smoke test through the `vllm-engine-core-client` utility interface:
```bash
cargo run -p vllm-engine-core-client --example external_engine_utility_call -- \
--handshake-address tcp://127.0.0.1:62100 \
--host 127.0.0.1
```
If your current engine setup does not support sleep mode, skip the `sleep` / `wake_up` part of the
smoke:
```bash
cargo run -p vllm-engine-core-client --example external_engine_utility_call -- \
--handshake-address tcp://127.0.0.1:62100 \
--host 127.0.0.1 \
--skip-sleep-wake
```
Run the Rust smoke test for sample logprobs decoding through the raw engine-core request path:
```bash
cargo run -p vllm-engine-core-client --example external_engine_logprobs -- \
--handshake-address tcp://127.0.0.1:62100 \
--host 127.0.0.1
```
This smoke requests a small generated-token `logprobs` payload plus prompt logprobs over a much
longer prompt, so it exercises both the inline and aux-frame decode paths against a real engine.
The Rust client decodes those payloads into semantic per-position records rather than exposing the
raw ndarray/tensor wire shape.
IMPORTANT: You must restart `vllm` each time you run the smoke test, as the vLLM engine cannot manage frontend closures and subsequent reconnects. In other words, do not reuse existing `vllm` instances, if any.
@@ -0,0 +1,190 @@
use std::time::Duration;
use anyhow::{Context, Result, bail};
use clap::Parser;
use futures::StreamExt as _;
use tokio::time::timeout;
use tracing_subscriber::EnvFilter;
use vllm_engine_core_client::protocol::{
EngineCoreFinishReason, EngineCoreRequest, EngineCoreSamplingParams,
};
use vllm_engine_core_client::{
EngineCoreClient, EngineCoreClientConfig, EngineCoreStreamOutput, TransportMode,
};
const BASE_PROMPT_TOKEN_IDS: &[u32] = &[20841, 448, 6896, 25, 23811];
#[derive(Debug, Parser)]
#[command(about = "Smoke-test engine-core sample logprobs against an external vLLM engine.")]
struct Args {
#[arg(long)]
handshake_address: String,
#[arg(long, default_value_t = 1)]
engine_count: usize,
#[arg(long, default_value = "Qwen/Qwen3-0.6B")]
model: String,
#[arg(long, default_value = "127.0.0.1")]
host: String,
#[arg(long, default_value_t = 0)]
client_index: u32,
#[arg(long, default_value_t = 30)]
ready_timeout_secs: u64,
#[arg(long, default_value_t = 120)]
output_timeout_secs: u64,
#[arg(long, default_value_t = 1)]
max_tokens: u32,
#[arg(long, default_value_t = 2)]
logprobs: i32,
#[arg(long, default_value_t = 1)]
prompt_logprobs: i32,
#[arg(long, default_value_t = 96)]
prompt_repeats: usize,
}
fn init_tracing() {
let filter = EnvFilter::try_from_default_env()
.unwrap_or_else(|_| EnvFilter::new("vllm_engine_core_client=debug"));
let _ = tracing_subscriber::fmt().with_env_filter(filter).try_init();
}
fn unique_request_id() -> String {
let nanos = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.expect("system clock should be after unix epoch")
.as_nanos();
format!("rust-engine-core-logprobs-{nanos}")
}
fn build_prompt_token_ids(prompt_repeats: usize) -> Vec<u32> {
let repeats = prompt_repeats.max(1);
BASE_PROMPT_TOKEN_IDS.repeat(repeats)
}
fn build_request(
request_id: String,
prompt_token_ids: Vec<u32>,
max_tokens: u32,
logprobs: i32,
prompt_logprobs: i32,
client_index: u32,
) -> EngineCoreRequest {
EngineCoreRequest {
request_id,
prompt_token_ids: Some(prompt_token_ids),
sampling_params: Some(EngineCoreSamplingParams {
max_tokens,
logprobs: Some(logprobs),
prompt_logprobs: Some(prompt_logprobs),
..EngineCoreSamplingParams::for_test()
}),
arrival_time: 0.0,
client_index,
..EngineCoreRequest::default()
}
}
async fn wait_for_final_output(
mut stream: vllm_engine_core_client::EngineCoreOutputStream,
) -> Result<EngineCoreStreamOutput> {
while let Some(output) = stream.next().await {
let output = output.context("failed to receive engine-core output")?;
if output.finished() {
return Ok(output);
}
}
bail!("request stream ended without a final output")
}
#[tokio::main(flavor = "multi_thread")]
async fn main() -> Result<()> {
init_tracing();
let args = Args::parse();
let ready_timeout = Duration::from_secs(args.ready_timeout_secs);
let output_timeout = Duration::from_secs(args.output_timeout_secs);
let request_id = unique_request_id();
let prompt_token_ids = build_prompt_token_ids(args.prompt_repeats);
let client = EngineCoreClient::connect(EngineCoreClientConfig {
transport_mode: TransportMode::HandshakeOwner {
handshake_address: args.handshake_address.clone(),
advertised_host: args.host.clone(),
engine_count: args.engine_count,
ready_timeout,
local_input_address: None,
local_output_address: None,
},
coordinator_mode: None,
model_name: args.model.clone(),
client_index: args.client_index,
})
.await
.context("failed to connect to external vLLM engine")?;
println!("model={}", args.model);
println!("handshake_address={}", args.handshake_address);
println!("engine_count={}", args.engine_count);
println!("input_address={}", client.input_address());
println!("output_address={}", client.output_address());
println!("engine_identities={:x?}", client.engine_identities());
let request = build_request(
request_id.clone(),
prompt_token_ids.clone(),
args.max_tokens,
args.logprobs,
args.prompt_logprobs,
args.client_index,
);
println!("request_id={request_id}");
println!("prompt_len={}", prompt_token_ids.len());
println!("base_prompt_len={}", BASE_PROMPT_TOKEN_IDS.len());
println!("prompt_repeats={}", args.prompt_repeats);
println!("requested_logprobs={}", args.logprobs);
println!("requested_prompt_logprobs={}", args.prompt_logprobs);
let stream = client.call(request).await.context("failed to submit engine-core request")?;
let output = timeout(output_timeout, wait_for_final_output(stream))
.await
.context("timed out waiting for final output")??;
let finish_reason = output.finish_reason;
let token_ids = output.new_token_ids.clone();
let logprobs = output
.new_logprobs
.as_ref()
.and_then(|value| value.as_direct())
.context("engine output did not include decoded sample logprobs")?;
let prompt_logprobs = output
.new_prompt_logprobs_tensors
.as_ref()
.and_then(|value| value.as_direct())
.context("engine output did not include decoded prompt logprobs")?;
println!("token_ids={token_ids:?}");
println!("finish_reason={finish_reason:?}");
println!("new_logprobs={logprobs:#?}");
println!("new_prompt_logprobs_tensors={prompt_logprobs:#?}");
client.shutdown().await.context("failed to shut down engine-core client")?;
if finish_reason != Some(EngineCoreFinishReason::Length) {
bail!("unexpected finish_reason: expected Length, got {finish_reason:?}");
}
if token_ids.is_empty() {
bail!("engine returned no generated token ids");
}
if logprobs.is_empty() {
bail!("decoded logprobs payload is unexpectedly empty");
}
if prompt_logprobs.is_empty() {
bail!("decoded prompt logprobs payload is unexpectedly empty");
}
if prompt_logprobs.len() + 1 < prompt_token_ids.len() {
bail!(
"prompt logprobs rows look too short: prompt_len={}, rows={}",
prompt_token_ids.len(),
prompt_logprobs.len()
);
}
Ok(())
}
@@ -0,0 +1,143 @@
use std::time::Duration;
use anyhow::{Context, Result, bail};
use clap::Parser;
use tracing_subscriber::EnvFilter;
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, TransportMode};
#[derive(Debug, Parser)]
#[command(about = "Smoke-test EngineCoreClient utility calls against an external vLLM engine.")]
struct Args {
#[arg(long)]
handshake_address: String,
#[arg(long, default_value_t = 1)]
engine_count: usize,
#[arg(long, default_value = "Qwen/Qwen3-0.6B")]
model: String,
#[arg(long, default_value = "127.0.0.1")]
host: String,
#[arg(long, default_value_t = 0)]
client_index: u32,
#[arg(long, default_value_t = 30)]
ready_timeout_secs: u64,
#[arg(
long,
default_value_t = false,
help = "Expected initial result of is_sleeping() before running the smoke steps."
)]
expected_is_sleeping: bool,
#[arg(long, default_value_t = false)]
reset_running_requests: bool,
#[arg(long, default_value_t = false)]
reset_external: bool,
#[arg(long, default_value_t = 1)]
sleep_level: u32,
#[arg(long, default_value = "abort")]
sleep_mode: String,
#[arg(
long,
default_value_t = false,
help = "Skip sleep/wake_up calls when the engine was not started with sleep-mode support."
)]
skip_sleep_wake: bool,
}
fn init_tracing() {
let filter = EnvFilter::try_from_default_env()
.unwrap_or_else(|_| EnvFilter::new("vllm_engine_core_client=debug"));
let _ = tracing_subscriber::fmt().with_env_filter(filter).try_init();
}
#[tokio::main(flavor = "multi_thread")]
async fn main() -> Result<()> {
init_tracing();
let args = Args::parse();
let client = EngineCoreClient::connect(EngineCoreClientConfig {
transport_mode: TransportMode::HandshakeOwner {
handshake_address: args.handshake_address.clone(),
advertised_host: args.host.clone(),
engine_count: args.engine_count,
ready_timeout: Duration::from_secs(args.ready_timeout_secs),
local_input_address: None,
local_output_address: None,
},
coordinator_mode: None,
model_name: args.model.clone(),
client_index: args.client_index,
})
.await
.context("failed to connect to external vLLM engine")?;
println!("model={}", args.model);
println!("handshake_address={}", args.handshake_address);
println!("engine_count={}", args.engine_count);
println!("input_address={}", client.input_address());
println!("output_address={}", client.output_address());
println!("engine_identities={:x?}", client.engine_identities());
let initial_is_sleeping =
client.is_sleeping().await.context("failed to call is_sleeping utility")?;
println!("is_sleeping(initial)={initial_is_sleeping}");
if initial_is_sleeping != args.expected_is_sleeping {
bail!(
"unexpected initial is_sleeping state: expected {}, got {}",
args.expected_is_sleeping,
initial_is_sleeping
);
}
let reset_prefix_cache = client
.reset_prefix_cache(args.reset_running_requests, args.reset_external)
.await
.context("failed to call reset_prefix_cache utility")?;
println!("reset_prefix_cache={reset_prefix_cache}");
client.reset_mm_cache().await.context("failed to call reset_mm_cache utility")?;
println!("reset_mm_cache=ok");
client
.reset_encoder_cache()
.await
.context("failed to call reset_encoder_cache utility")?;
println!("reset_encoder_cache=ok");
if args.skip_sleep_wake {
println!("sleep_wake=skipped");
} else {
client.sleep(args.sleep_level, &args.sleep_mode).await.with_context(|| {
format!(
"failed to call sleep utility with level={} mode={}",
args.sleep_level, args.sleep_mode
)
})?;
println!(
"sleep=ok level={} mode={}",
args.sleep_level, args.sleep_mode
);
let sleeping_after_sleep =
client.is_sleeping().await.context("failed to call is_sleeping after sleep")?;
println!("is_sleeping(after_sleep)={sleeping_after_sleep}");
if !sleeping_after_sleep {
bail!("engine should report sleeping=true after sleep()");
}
client.wake_up(None).await.context("failed to call wake_up utility")?;
println!("wake_up=ok");
let sleeping_after_wake =
client.is_sleeping().await.context("failed to call is_sleeping after wake_up")?;
println!("is_sleeping(after_wake)={sleeping_after_wake}");
if sleeping_after_wake {
bail!("engine should report sleeping=false after wake_up()");
}
}
client.shutdown().await.context("failed to shut down engine-core client")?;
Ok(())
}
+673
View File
@@ -0,0 +1,673 @@
use std::sync::Arc;
use std::time::Duration;
use futures::future::{join_all, try_join_all};
use tokio::sync::mpsc;
use tokio_util::task::AbortOnDropHandle;
use tracing::{debug, info, trace};
use crate::client::imp::{ClientInner, run_abort_loop, run_output_dispatcher_loop};
use crate::coordinator::CoordinatorHandle;
use crate::error::{Error, Result};
use crate::protocol::handshake::EngineCoreReadyResponse;
use crate::protocol::{
EngineCoreRequest, EngineCoreRequestType, EngineCoreUtilityRequest, ModelDtype,
};
use crate::transport::{self, ConnectedEngine};
pub(crate) mod imp;
mod state;
mod stream;
pub use stream::{EngineCoreOutputStream, EngineCoreStreamOutput};
/// How the frontend acquires its request/response transport with Python
/// `EngineCoreProc`s.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum TransportMode {
/// The Rust process owns the startup handshake and allocates or binds the
/// frontend transport addresses itself before replying to engine
/// `HELLO` messages.
HandshakeOwner {
/// Shared handshake endpoint that engines dial during startup.
handshake_address: String,
/// Host/IP that engines should use to connect back to the frontend
/// transport sockets.
advertised_host: String,
/// Total number of engines expected to join this transport.
engine_count: usize,
/// Maximum time to wait for each startup phase to complete.
ready_timeout: Duration,
/// Optional explicit bind address for the input ROUTER socket.
local_input_address: Option<String>,
/// Optional explicit bind address for the output PULL socket.
local_output_address: Option<String>,
},
/// The Python supervisor has already chosen the frontend transport
/// addresses, and the Rust process only needs to bind them and wait for
/// engine registration frames.
Bootstrapped {
/// Input ROUTER socket address that engines will connect to for
/// requests.
input_address: String,
/// Output PULL socket address that engines will connect to for
/// responses.
output_address: String,
/// Total number of engines expected to register on this transport.
engine_count: usize,
/// Maximum time to wait for all expected engines to register.
ready_timeout: Duration,
},
}
/// Which coordinator implementation should be active when one is present for a
/// frontend client.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum CoordinatorMode {
/// Run the Rust in-process coordinator for managed `serve` deployments.
InProc,
/// Connect to an external coordinator owned by another process.
External { address: String },
}
/// Configuration for connecting a Rust frontend client to an already running
/// Python `EngineCoreProc`.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct EngineCoreClientConfig {
/// Frontend-to-engine transport setup.
pub transport_mode: TransportMode,
/// Frontend-side coordinator behavior, or `None` when requests should flow
/// directly to engines without any coordinator involvement.
pub coordinator_mode: Option<CoordinatorMode>,
/// Model name used for frontend-side metrics labels.
pub model_name: String,
/// Frontend client index stamped onto every request.
pub client_index: u32,
}
impl EngineCoreClientConfig {
/// Create a new client config with the given handshake address, expecting a
/// single engine, and default values for all other fields.
pub fn new_single(handshake_address: impl Into<String>) -> Self {
Self {
transport_mode: TransportMode::HandshakeOwner {
handshake_address: handshake_address.into(),
advertised_host: "127.0.0.1".to_string(),
engine_count: 1,
ready_timeout: Duration::from_secs(30),
local_input_address: None,
local_output_address: None,
},
coordinator_mode: None,
model_name: String::new(),
client_index: 0,
}
}
/// Set the model name used by frontend-side metrics and diagnostics.
pub fn with_model_name(mut self, model_name: impl Into<String>) -> Self {
self.model_name = model_name.into();
self
}
/// Override the client index stamped onto every outgoing request.
pub fn with_client_index(mut self, client_index: u32) -> Self {
self.client_index = client_index;
self
}
/// Override the optional coordinator mode for this client config.
pub fn with_coordinator_mode(mut self, coordinator_mode: Option<CoordinatorMode>) -> Self {
self.coordinator_mode = coordinator_mode;
self
}
/// Override the locally bound input/output addresses for handshake-owned
/// transport mode.
///
/// This is primarily used by tests that want deterministic IPC endpoints
/// while still exercising the handshake-owned startup path.
pub fn with_local_input_output_addresses(
mut self,
local_input_address: Option<String>,
local_output_address: Option<String>,
) -> Self {
let TransportMode::HandshakeOwner {
local_input_address: current_input,
local_output_address: current_output,
..
} = &mut self.transport_mode
else {
panic!("local input/output overrides are only valid in handshake-owned mode");
};
*current_input = local_input_address;
*current_output = local_output_address;
self
}
}
/// The reason a request stream is being aborted when its output stream is
/// dropped.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum AbortCause {
/// The consumer dropped the stream before the request reached a terminal
/// engine output.
#[default]
DroppedStream,
/// The frontend matched a stop string locally and intentionally stopped
/// consuming the stream.
StopStringMatched,
}
task_local::task_local! {
static ABORT_CAUSE: AbortCause;
}
impl AbortCause {
/// Return the abort cause currently associated with this task, or
/// [`AbortCause::DroppedStream`] by default.
pub fn current() -> Self {
ABORT_CAUSE.try_get().unwrap_or_default()
}
/// Drop one value while marking the drop as happening for this abort cause.
pub fn drop_as<T>(self, value: T) {
ABORT_CAUSE.sync_scope(self, move || drop(value));
}
}
/// Internal auto-abort work item sent from stream `Drop` handlers to the abort
/// worker.
#[derive(Debug, Clone)]
pub(crate) struct AbortRequest {
request_id: String,
cause: AbortCause,
}
/// Default ZMQ-based implementation that talks directly to a Python
/// `EngineCoreProc`.
pub struct EngineCoreClient {
config: EngineCoreClientConfig,
input_address: String,
output_address: String,
engines: Vec<ConnectedEngine>,
inner: Arc<ClientInner>,
coordinator: Option<CoordinatorHandle>,
abort_tx: mpsc::UnboundedSender<AbortRequest>,
// Background tasks
output_task: AbortOnDropHandle<()>,
dispatcher_task: AbortOnDropHandle<()>,
abort_task: AbortOnDropHandle<()>,
coordinator_output_task: Option<AbortOnDropHandle<()>>,
coordinator_task: Option<AbortOnDropHandle<()>>,
}
impl EngineCoreClient {
/// Connect to Python `EngineCoreProc`s using the configured
/// transport/coordinator modes.
///
/// In handshake-owned mode this method drives the full engine startup
/// handshake. In bootstrapped mode it binds the provided frontend
/// sockets and waits for the expected engine registration frames.
pub async fn connect(config: EngineCoreClientConfig) -> Result<Self> {
let connected = match &config.transport_mode {
TransportMode::HandshakeOwner {
handshake_address,
advertised_host,
engine_count,
ready_timeout,
local_input_address,
local_output_address,
} => {
let enable_inproc_coordinator = match config.coordinator_mode {
None => false,
Some(CoordinatorMode::InProc) => true,
Some(CoordinatorMode::External { .. }) => {
return Err(Error::UnsupportedExternalCoordinator);
}
};
transport::connect_handshake(
handshake_address,
*engine_count,
advertised_host,
local_input_address.as_deref(),
local_output_address.as_deref(),
enable_inproc_coordinator,
*ready_timeout,
)
.await?
}
TransportMode::Bootstrapped {
input_address,
output_address,
engine_count,
ready_timeout,
} => {
if let Some(CoordinatorMode::InProc) = config.coordinator_mode {
panic!("cannot use in-process coordinator with bootstrapped transport mode")
}
transport::connect_bootstrapped(
input_address,
output_address,
*engine_count,
*ready_timeout,
)
.await?
}
};
Self::from_connected(config, connected).await
}
/// Create a new client instance from the connected transport state after
/// the startup handshake completes.
async fn from_connected(
config: EngineCoreClientConfig,
connected: transport::ConnectedTransport,
) -> Result<Self> {
let (output_tx, output_rx) = mpsc::channel(64);
let (abort_tx, abort_rx) = mpsc::unbounded_channel();
let engines = connected.engines;
let inner = Arc::new(ClientInner::new(
connected.input_send,
config.model_name.clone(),
&engines,
));
let output_task = AbortOnDropHandle::new(tokio::spawn(transport::run_output_loop(
connected.output_socket,
output_tx,
)));
let dispatcher_task = AbortOnDropHandle::new(tokio::spawn(run_output_dispatcher_loop(
inner.clone(),
output_rx,
)));
let abort_task =
AbortOnDropHandle::new(tokio::spawn(run_abort_loop(inner.clone(), abort_rx)));
// If any engine reported a dp_stats_address in its ready response, use it
// as the external coordinator address.
let dp_stats_address: Option<String> = engines
.iter()
.filter_map(|e| e.ready_response.as_ref())
.find_map(|r| r.dp_stats_address.clone());
let (coordinator, coordinator_output_task, coordinator_task) =
if let Some(coordinator_transport) = connected.coordinator {
let (handle, runner) =
CoordinatorHandle::new_inproc(coordinator_transport.input_socket);
let (coordinator_output_tx, coordinator_output_rx) = mpsc::channel(64);
let coordinator_output_task =
AbortOnDropHandle::new(tokio::spawn(transport::run_output_loop(
coordinator_transport.output_socket,
coordinator_output_tx,
)));
let coordinator_task = AbortOnDropHandle::new(tokio::spawn(
runner.run(coordinator_output_rx, inner.clone()),
));
(
Some(handle),
Some(coordinator_output_task),
Some(coordinator_task),
)
} else if let Some(address) =
dp_stats_address.as_deref().or(match config.coordinator_mode.as_ref() {
Some(CoordinatorMode::External { address }) => Some(address.as_str()),
_ => None,
})
{
let (handle, service) = CoordinatorHandle::connect_external(address).await?;
let coordinator_task =
AbortOnDropHandle::new(tokio::spawn(service.run(inner.clone())));
(Some(handle), None, Some(coordinator_task))
} else {
(None, None, None)
};
Ok(Self {
config,
input_address: connected.input_address,
output_address: connected.output_address,
engines,
inner,
coordinator,
abort_tx,
output_task,
dispatcher_task,
abort_task,
coordinator_output_task,
coordinator_task,
})
}
/// Return the address of the input socket that the client uses to send
/// requests to the engine.
pub fn input_address(&self) -> &str {
&self.input_address
}
/// Return the address of the output socket that the client listens on for
/// engine responses.
pub fn output_address(&self) -> &str {
&self.output_address
}
/// Return the number of engines connected to this client.
pub fn engine_count(&self) -> usize {
self.engines.len()
}
/// Return the engine identities of all engines connected to this client.
pub fn engine_identities(&self) -> Vec<&[u8]> {
self.engines.iter().map(|engine| &*engine.engine_id).collect()
}
/// Return the ready responses received from all engines on the input
/// socket.
pub fn ready_responses(&self) -> Vec<&EngineCoreReadyResponse> {
self.engines
.iter()
.filter_map(|engine| engine.ready_response.as_ref())
.collect()
}
/// Return the engine-reported effective model dtype, when available.
pub fn model_dtype(&self) -> Option<ModelDtype> {
self.engines
.iter()
.filter_map(|engine| engine.ready_response.as_ref())
.find_map(|response| response.dtype)
}
/// Return the total number of GPU blocks summed across all connected
/// engines.
pub fn total_num_gpu_blocks(&self) -> u64 {
self.engines
.iter()
.filter_map(|engine| engine.ready_response.as_ref())
.map(|r| r.num_gpu_blocks)
.sum()
}
/// Return the minimum engine-reported `max_model_len` across all engines.
///
/// This is the auto-fitted value after KV cache profiling and may differ
/// from the originally configured value.
pub fn max_model_len(&self) -> Option<u32> {
self.engines
.iter()
.filter_map(|e| e.ready_response.as_ref())
.map(|r| r.max_model_len as u32)
.min()
}
/// Get the model name associated with this client used for metrics
/// labeling.
pub fn model_name(&self) -> &str {
self.inner.model_name()
}
/// Return whether the client still considers the engine healthy.
pub fn is_healthy(&self) -> bool {
self.inner.is_healthy()
}
/// Return the first persistent health error observed by the client, if any.
pub fn health_error(&self) -> Option<Arc<Error>> {
self.inner.health_error()
}
}
// Client API implementation.
impl EngineCoreClient {
/// Add a new request to the engine and return a per-request raw output
/// stream.
pub async fn call(&self, mut req: EngineCoreRequest) -> Result<EngineCoreOutputStream> {
req.client_index = self.config.client_index;
req.validate()?;
trace!(
request_id = %req.request_id,
client_index = req.client_index,
current_wave = req.current_wave,
request = ?req,
"sending add request"
);
let request_id = req.request_id.clone();
let data_parallel_rank = req.data_parallel_rank;
let (engine_id, rx) =
self.inner.register_request(request_id.clone(), data_parallel_rank)?;
let result: Result<()> = async {
if let Some(coordinator) = self.coordinator.as_ref() {
let snapshot = coordinator.snapshot();
req.current_wave = snapshot.current_wave;
if !snapshot.engines_running {
coordinator.notify_first_request(engine_id.clone())?;
}
}
debug!(
request_id = req.request_id,
?engine_id,
"registered request to engine"
);
self.inner.send_to_engine(&engine_id, EngineCoreRequestType::Add, &req).await?;
Ok(())
}
.await;
// Failed to send the request to the engine, roll back the registration.
if let Err(error) = result {
self.inner.rollback_request(&request_id);
return Err(error);
}
Ok(EngineCoreOutputStream::new(
request_id,
self.abort_tx.clone(),
rx,
))
}
/// Abort currently in-flight requests by request ID.
pub async fn abort(&self, ids: &[String]) -> Result<()> {
let abortable = self.inner.abortable_request_ids(ids)?;
trace!(request_ids = ?ids, abortable_request_ids = ?abortable, "sending abort request ids");
if abortable.is_empty() {
return Ok(());
}
for (engine_id, request_ids) in abortable {
self.inner.do_abort_requests(&engine_id, &request_ids).await?;
}
Ok(())
}
/// Call a typed utility method on all connected engines, returning one
/// decoded result per connected engine if all calls succeed or an error
/// if any call fails.
///
/// Callers should pass utility arguments using Rust tuple semantics so the
/// encoded payload matches Python's `(client_index, call_id,
/// method_name, args)` contract: `()`, `(arg,)`, `(arg1, arg2)`, etc.
pub async fn call_utility<T, A>(&self, method: &str, args: A) -> Result<Vec<T>>
where
T: serde::de::DeserializeOwned,
A: serde::Serialize + std::fmt::Debug,
{
trace!(
method,
client_index = self.config.client_index,
engine_count = self.engines.len(),
"sending utility request"
);
// Phase 1: allocate one call id per engine and build the per-engine
// request payloads up-front. Any failure here (registry closed, encode
// error) must roll back the call ids already allocated so they do not
// leak in the utility registry until shutdown.
let mut pending_calls = Vec::with_capacity(self.engines.len());
let mut prepared_sends = Vec::with_capacity(self.engines.len());
for engine in &self.engines {
let (call_id, rx) = match self.inner.allocate_and_register_utility_call() {
Ok(pair) => pair,
Err(err) => {
self.inner.unregister_utility_calls(pending_calls.iter().map(|(id, _)| *id));
return Err(err);
}
};
let request = match EngineCoreUtilityRequest::new(
self.config.client_index,
call_id,
method,
&args,
) {
Ok(request) => request,
Err(err) => {
self.inner.unregister_utility_calls(
pending_calls.iter().map(|(id, _)| *id).chain(std::iter::once(call_id)),
);
return Err(err);
}
};
pending_calls.push((call_id, rx));
prepared_sends.push((&engine.engine_id, request));
}
// Phase 2: dispatch every utility request concurrently. `try_join_all`
// fails fast on the first transport error and drops the remaining send
// futures; any engines that already received the request will reply,
// but those replies are simply dropped because we roll back the call
// ids below.
let send_futures = prepared_sends.iter().map(|(engine_id, request)| {
self.inner.send_to_engine(engine_id, EngineCoreRequestType::Utility, request)
});
if let Err(err) = try_join_all(send_futures).await {
self.inner.unregister_utility_calls(pending_calls.iter().map(|(id, _)| *id));
return Err(err);
}
// Phase 3: wait for all engines to respond and preserve the per-engine
// result list.
let futures = pending_calls.into_iter().map(|(call_id, rx)| async move {
rx.await
.map_err(|_| Error::UtilityCallClosed {
method: method.to_string(),
call_id,
})??
.into_typed_result(method)
});
try_join_all(futures).await
}
/// Execute `collective_rpc` on all engines and flatten all engine results
/// into one list.
pub async fn collective_rpc<A, K>(
&self,
method: &str,
timeout: Option<f64>,
args: A,
kwargs: K,
) -> Result<Vec<rmpv::Value>>
where
A: serde::Serialize + std::fmt::Debug,
K: serde::Serialize + std::fmt::Debug,
{
let results = self
.call_utility::<rmpv::Value, _>("collective_rpc", (method, timeout, args, kwargs))
.await?;
Ok(results
.into_iter()
.flat_map(|result| match result {
// Each engine's `collective_rpc` result is itself the worker-level result list.
rmpv::Value::Array(results) => results,
other => vec![other],
})
.collect())
}
/// Return whether the engine is currently sleeping at any level.
pub async fn is_sleeping(&self) -> Result<bool> {
// TODO: we only return the result of the first engine here.
Ok(self.call_utility("is_sleeping", ()).await?[0])
}
/// Reset the multi-modal cache.
pub async fn reset_mm_cache(&self) -> Result<()> {
self.call_utility::<(), _>("reset_mm_cache", ()).await?;
Ok(())
}
/// Reset the encoder cache.
pub async fn reset_encoder_cache(&self) -> Result<()> {
self.call_utility::<(), _>("reset_encoder_cache", ()).await?;
Ok(())
}
/// Reset the prefix cache and optionally the external connector cache.
pub async fn reset_prefix_cache(
&self,
reset_running_requests: bool,
reset_connector: bool,
) -> Result<bool> {
// TODO: we only return the result of the first engine here.
Ok(self
.call_utility(
"reset_prefix_cache",
(reset_running_requests, reset_connector),
)
.await?[0])
}
/// Put the engine to sleep.
pub async fn sleep(&self, level: u32, mode: &str) -> Result<()> {
self.call_utility::<(), _>("sleep", (level, mode)).await?;
Ok(())
}
/// Wake the engine from sleep, optionally limiting the wake-up to specific
/// tags.
pub async fn wake_up(&self, tags: Option<Vec<String>>) -> Result<()> {
self.call_utility::<(), _>("wake_up", (tags,)).await?;
Ok(())
}
/// Shut down local client tasks and close transport state.
pub async fn shutdown(self) -> Result<()> {
let Self {
inner,
abort_tx,
output_task,
dispatcher_task,
abort_task,
coordinator_output_task,
coordinator_task,
..
} = self;
info!("shutting down engine-core client");
inner.shutdown();
drop(abort_tx);
// Abort all client tasks first, then await them.
// Note the aborting orders here.
let mut tasks = vec![abort_task, dispatcher_task, output_task];
tasks.extend(coordinator_task);
tasks.extend(coordinator_output_task);
tasks.iter().for_each(|t| t.abort());
join_all(tasks).await;
info!("engine-core client shut down");
Ok(())
}
}
@@ -0,0 +1,410 @@
use std::collections::BTreeMap;
use std::slice;
use std::sync::Arc;
use arc_swap::ArcSwapOption;
use parking_lot::Mutex;
use thiserror_ext::AsReport as _;
use tokio::sync::mpsc;
use tracing::{debug, info, trace, warn};
use vllm_metrics::METRICS;
use zeromq::RouterSendHalf;
use crate::client::state::{OutputReceiver, RequestRegistry, UtilityReceiver, UtilityRegistry};
use crate::client::stream::EngineCoreStreamOutput;
use crate::client::{AbortCause, AbortRequest};
use crate::error::{client_closed, dispatcher_closed, unexpected_dispatcher_output};
use crate::metrics::record_scheduler_stats;
use crate::protocol::stats::SchedulerStats;
use crate::protocol::{
ClassifiedEngineCoreOutputs, EngineCoreOutput, EngineCoreOutputs, EngineCoreRequestType,
UtilityOutput, encode_msgpack,
};
use crate::transport::{ConnectedEngine, EngineId};
use crate::{Error, Result, transport};
pub(crate) struct ClientInner {
input_send: RouterSendHalf,
model_name: String,
request_reg: Mutex<RequestRegistry>,
utility_reg: Mutex<UtilityRegistry>,
health_error: ArcSwapOption<Error>,
}
impl ClientInner {
/// Create a new instance with the given input send half after the startup
/// handshake completes.
pub fn new(
input_send: RouterSendHalf,
model_name: String,
engines: &[ConnectedEngine],
) -> Self {
Self {
input_send,
model_name,
request_reg: Mutex::new(RequestRegistry::new(engines)),
utility_reg: Mutex::new(UtilityRegistry::default()),
health_error: ArcSwapOption::empty(),
}
}
/// Get the model name associated with this client used for metrics
/// labeling.
pub fn model_name(&self) -> &str {
&self.model_name
}
/// Register a newly added request. Return the selected engine id and the
/// per-request output channel bound to its `request_id`.
///
/// When `data_parallel_rank` is provided, the request is routed to that
/// specific engine rank, bypassing load balancing.
pub fn register_request(
&self,
request_id: String,
data_parallel_rank: Option<u32>,
) -> Result<(EngineId, OutputReceiver)> {
let mut registry = self.request_reg.lock();
if registry.is_closed() {
return Err(self.closed_error());
}
registry.register(request_id, data_parallel_rank)
}
/// Allocate the next utility `call_id` and register its waiting receiver.
pub fn allocate_and_register_utility_call(&self) -> Result<(i64, UtilityReceiver)> {
let mut registry = self.utility_reg.lock();
if registry.is_closed() {
return Err(self.closed_error());
}
Ok(registry.allocate_and_register())
}
/// Undo a batch of utility call allocations when the fan-out send fails
/// partway through. Silently ignores unknown call ids so callers can pass
/// the full set without first filtering successful sends.
pub fn unregister_utility_calls(&self, call_ids: impl IntoIterator<Item = i64>) {
self.utility_reg.lock().unregister_many(call_ids);
}
/// Undo a request registration when `add_request()` fails.
pub fn rollback_request(&self, request_id: &str) {
let _ = self.request_reg.lock().remove(request_id);
}
/// Filter the given request IDs to the subset that are still tracked as
/// active and can be aborted, grouped by the engine that originally
/// accepted them.
pub fn abortable_request_ids(
&self,
request_ids: &[String],
) -> Result<BTreeMap<EngineId, Vec<String>>> {
let registry = self.request_reg.lock();
if registry.is_closed() {
return Err(self.closed_error());
}
Ok(registry.abortable_request_ids(request_ids))
}
/// Obtain the stream sender for one output. If it indicates the request is
/// finished, it will be removed from the registry.
pub fn take_sender_for_output(
&self,
output: &EngineCoreOutput,
) -> Option<mpsc::UnboundedSender<Result<EngineCoreStreamOutput>>> {
self.request_reg.lock().sender_for_output(output)
}
/// Remove a batch of requests that have finished or aborted, returning
/// their stream senders.
pub fn finish_requests<'a>(
&self,
request_ids: impl IntoIterator<Item = &'a String>,
) -> Vec<mpsc::UnboundedSender<Result<EngineCoreStreamOutput>>> {
self.request_reg.lock().finish_many(request_ids)
}
/// Apply one scheduler stats update for the given engine to the local
/// routing state. Returns `false` if the engine is unknown to the
/// client.
pub fn apply_scheduler_stats(&self, engine_index: u32, stats: &SchedulerStats) -> bool {
self.request_reg.lock().apply_scheduler_stats(engine_index, stats)
}
/// Close all active request streams and utility calls with the first
/// persistent health error.
pub fn close_registries(&self, error: Arc<Error>) {
let persistent_error = self.record_health_error(error);
let request_senders = self.request_reg.lock().close();
let utility_senders = self.utility_reg.lock().close();
// Notify all ongoing requests that the client is closed.
for sender in request_senders {
let _ = sender.send(Err(Error::Shared(persistent_error.clone())));
}
for sender in utility_senders {
let _ = sender.send(Err(Error::Shared(persistent_error.clone())));
}
}
/// Return the first persistent health error observed by the client, if any.
pub fn health_error(&self) -> Option<Arc<Error>> {
self.health_error.load_full()
}
/// Return whether the client still considers the engine healthy.
pub fn is_healthy(&self) -> bool {
self.health_error.load().is_none()
}
/// Resolve one utility output to the waiting caller. Returns `true` if a
/// waiting caller existed.
pub fn resolve_utility_output(&self, output: UtilityOutput) -> bool {
match self.utility_reg.lock().resolve(&output.call_id) {
Some(sender) => {
sender.send(Ok(output)).unwrap_or_default();
true
}
None => false,
}
}
/// Send the given message to the engine. The request should be first
/// registered via `register_request()` to ensure the request stream is
/// tracked.
pub async fn send_to_engine<T>(
&self,
engine_id: &EngineId,
request_type: EngineCoreRequestType,
payload: &T,
) -> Result<()>
where
T: serde::Serialize + std::fmt::Debug,
{
// TODO: for `EngineCoreRequest`, split outbound tensor raw views into aux
// frames instead of always producing a single msgpack frame.
let payload = encode_msgpack(payload)?;
let mut input_send = self.input_send.clone();
transport::send_message(&mut input_send, engine_id, request_type.to_frame(), payload)
.await?;
Ok(())
}
/// Handle an abort request by sending the abort message to the engine.
pub async fn do_abort_requests(
&self,
engine_id: &EngineId,
request_ids: &[String],
) -> Result<()> {
self.send_to_engine(engine_id, EngineCoreRequestType::Abort, &request_ids).await
}
/// Shut down by closing all active request streams and utility calls with a
/// sticky client closed error.
pub fn shutdown(&self) {
self.close_registries(Arc::new(client_closed!("engine-core client shut down")));
}
/// Remove the request from the active registry for auto-abort and return
/// the engine that the request was originally routed to, if it is still
/// active.
pub fn take_auto_abort_target(&self, request_id: &str) -> Option<EngineId> {
let mut registry = self.request_reg.lock();
let (_, engine_id) = registry.remove(request_id)?;
if registry.is_closed() {
return None;
}
Some(engine_id)
}
/// Publish the first persistent health error and return the sticky error
/// recorded for this client. Later failures do not overwrite the first
/// one so `/health` and post-close callers observe a stable cause.
fn record_health_error(&self, error: Arc<Error>) -> Arc<Error> {
if let Some(existing) = self.health_error.load_full() {
return existing;
}
self.health_error
.rcu(|current| current.clone().unwrap_or_else(|| error.clone()));
self.health_error
.load_full()
.expect("health error must be recorded before registries close")
}
/// Assert there is a recorded health error and return a `Shared` variant
/// wrapping it for error returns when the client is already closed.
fn closed_error(&self) -> Error {
Error::Shared(self.health_error.load_full().expect(
"closed registry must have a recorded health error before rejecting new operations",
))
}
}
/// Background loop that listens for request IDs to abort and sends abort
/// messages to the engine. This is used to implement the auto-abort behavior
/// when a request stream is dropped without being properly terminated.
pub(crate) async fn run_abort_loop(
inner: Arc<ClientInner>,
mut abort_rx: mpsc::UnboundedReceiver<AbortRequest>,
) {
// TODO: receive and abort requests in batch
while let Some(AbortRequest { request_id, cause }) = abort_rx.recv().await {
let Some(engine_id) = inner.take_auto_abort_target(&request_id) else {
debug!(request_id, "skip auto-abort for inactive request");
continue;
};
match cause {
AbortCause::DroppedStream => {
info!(request_id, "auto-aborting request due to dropped stream")
}
AbortCause::StopStringMatched => {
debug!(
request_id,
"auto-aborting request due to stop string matched"
)
}
}
if let Err(error) = inner.do_abort_requests(&engine_id, slice::from_ref(&request_id)).await
{
warn!(
request_id,
?engine_id,
error = %error.as_report(),
"failed to auto-abort dropped request stream"
);
}
}
}
/// Background loop that listens for engine-core outputs and dispatches them to
/// the corresponding request streams based on their `request_id`.
pub(crate) async fn run_output_dispatcher_loop(
inner: Arc<ClientInner>,
mut output_rx: mpsc::Receiver<Result<EngineCoreOutputs>>,
) {
let result: Result<()> = async {
loop {
let outputs = match output_rx.recv().await {
Some(outputs) => outputs,
None => Err(dispatcher_closed!(
"engine-core output dispatcher channel closed"
)),
}?;
match outputs.classify() {
ClassifiedEngineCoreOutputs::RequestBatch(batch) => {
for output in batch.outputs {
let request_id = output.request_id.clone();
let Some(sender) = inner.take_sender_for_output(&output) else {
debug!(request_id, "dropping output for inactive request");
continue;
};
let wrapped_output = EngineCoreStreamOutput {
engine_index: batch.engine_index,
timestamp: batch.timestamp,
output,
};
if sender.send(Ok(wrapped_output)).is_err() {
debug!(request_id, "request output stream receiver dropped");
}
}
// The sender for normally-finished requests should have already been removed
// from the registry when their final output was dispatched
// above. This serves as a safety net to capture any
// requests marked as finished by the engine.
if let Some(finished_requests) = batch.finished_requests.as_ref() {
for request_id in finished_requests {
trace!(request_id, "request completed via finished_requests");
}
drop(inner.finish_requests(finished_requests));
}
if let Some(scheduler_stats) = batch.scheduler_stats.as_ref() {
if !inner.apply_scheduler_stats(batch.engine_index, scheduler_stats) {
debug!(
engine_index = batch.engine_index,
"dropping scheduler stats for unknown engine"
);
}
record_scheduler_stats(
&METRICS.scheduler,
inner.model_name(),
batch.engine_index,
scheduler_stats,
);
}
}
ClassifiedEngineCoreOutputs::Utility(utility) => {
let call_id = utility.output.call_id;
if inner.resolve_utility_output(utility.output) {
trace!(
call_id,
engine_index = utility.engine_index,
"resolved utility output"
);
} else {
warn!(
call_id,
engine_index = utility.engine_index,
"dropping output for unexpected utility call"
);
}
}
other @ (ClassifiedEngineCoreOutputs::DpControl { .. }
| ClassifiedEngineCoreOutputs::Other(_)) => {
Err::<(), _>(unexpected_dispatcher_output!(
"received unexpected output on main dispatcher path: {other:?}"
))?;
}
}
}
}
.await;
let Err(error) = result else { return };
warn!(error = %error.as_report(), "output dispatcher exiting with error");
inner.close_registries(Arc::new(error));
}
#[cfg(test)]
mod tests {
use zeromq::{RouterSocket, Socket};
use super::*;
async fn test_inner() -> ClientInner {
let mut socket = RouterSocket::new();
socket.bind("tcp://127.0.0.1:0").await.unwrap();
let (send, _) = socket.split();
ClientInner::new(
send,
"test-model".to_string(),
&[ConnectedEngine {
engine_id: EngineId::from(b"engine-0"),
ready_response: None,
}],
)
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn close_registries_records_first_health_error_only() {
let inner = test_inner().await;
inner.close_registries(Arc::new(Error::EngineCoreDead));
assert!(!inner.is_healthy());
assert!(matches!(
inner.health_error().as_deref(),
Some(Error::EngineCoreDead)
));
inner.close_registries(Arc::new(client_closed!("shutdown")));
assert!(matches!(
inner.health_error().as_deref(),
Some(Error::EngineCoreDead)
));
}
}
@@ -0,0 +1,670 @@
use std::collections::BTreeMap;
use std::sync::atomic::{AtomicI64, Ordering};
use tokio::sync::{mpsc, oneshot};
use tracing::trace;
use crate::EngineId;
use crate::client::stream::EngineCoreStreamOutput;
use crate::error::{Error, Result};
use crate::protocol::stats::SchedulerStats;
use crate::protocol::{EngineCoreOutput, UtilityOutput};
use crate::transport::ConnectedEngine;
pub type OutputSender = mpsc::UnboundedSender<Result<EngineCoreStreamOutput>>;
pub type OutputReceiver = mpsc::UnboundedReceiver<Result<EngineCoreStreamOutput>>;
pub type UtilitySender = oneshot::Sender<Result<UtilityOutput>>;
pub type UtilityReceiver = oneshot::Receiver<Result<UtilityOutput>>;
#[derive(Debug)]
struct TrackedRequest {
sender: OutputSender,
engine_id: EngineId,
}
/// The latest real scheduler-side load snapshot observed from one engine.
///
/// These counters come from `scheduler_stats` on the normal engine output path
/// and are the preferred routing signal once available.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct EngineLoadSnapshot {
/// Requests still counted on the scheduler's waiting side.
waiting: usize,
/// Requests currently counted on the scheduler's running side.
running: usize,
}
#[derive(Debug, Default)]
struct EngineRoutingState {
/// Requests admitted by this frontend that have not finished yet.
///
/// This is used both as the bootstrap fallback before real scheduler stats
/// exist and as a lower bound afterwards so asynchronous scheduler
/// snapshots cannot erase frontend admission history.
inflight: usize,
/// The latest real scheduler snapshot received from this engine, if any.
last_scheduler_stats: Option<EngineLoadSnapshot>,
}
impl EngineRoutingState {
/// Compute the routing score used to pick the least-loaded engine.
///
/// Scheduler stats can raise the load estimate above the frontend-local
/// view, but they should not lower it below requests this frontend has
/// already admitted. Waiting requests still get the same extra penalty
/// as the original `waiting * 4 + running` score.
fn routing_score(&self) -> usize {
const WAITING_WEIGHT: usize = 4;
let Some(stats) = self.last_scheduler_stats else {
return self.inflight;
};
let scheduler_total = stats.running + stats.waiting;
self.inflight.max(scheduler_total) + stats.waiting * (WAITING_WEIGHT - 1)
}
/// Replace the local routing view with a fresh real scheduler snapshot.
fn apply_scheduler_counts(&mut self, next: EngineLoadSnapshot) {
self.last_scheduler_stats = Some(next);
}
}
/// Internal registry for tracking active requests and their output stream
/// senders.
///
/// This is used to route incoming outputs to the correct request stream, and to
/// ensure proper cleanup of senders when requests finish or the client shuts
/// down.
#[derive(Debug)]
pub struct RequestRegistry {
closed: bool,
requests: BTreeMap<String, TrackedRequest>,
routing_per_engine: BTreeMap<EngineId, EngineRoutingState>,
}
impl RequestRegistry {
pub fn new(engines: &[ConnectedEngine]) -> Self {
Self {
closed: false,
requests: BTreeMap::default(),
routing_per_engine: engines
.iter()
.map(|engine| (engine.engine_id.clone(), EngineRoutingState::default()))
.collect(),
}
}
/// Register a newly added request. Create the per-request output channel
/// bound to its `request_id` and return the selected engine id.
///
/// When `data_parallel_rank` is provided, the request is routed directly to
/// the engine at that rank index, bypassing load balancing. Otherwise
/// the engine with the fewest in-flight requests is chosen.
pub fn register(
&mut self,
request_id: String,
data_parallel_rank: Option<u32>,
) -> Result<(EngineId, OutputReceiver)> {
if self.requests.contains_key(&request_id) {
return Err(Error::DuplicateRequestId { request_id });
}
let engine_id = self.choose_engine_for_request(data_parallel_rank)?;
let (tx, rx) = mpsc::unbounded_channel();
self.requests.insert(
request_id,
TrackedRequest {
sender: tx,
engine_id: engine_id.clone(),
},
);
let state = self
.routing_per_engine
.get_mut(&engine_id)
.expect("request registry must track all known engines");
state.inflight += 1;
Ok((engine_id, rx))
}
fn choose_engine_for_request(&mut self, data_parallel_rank: Option<u32>) -> Result<EngineId> {
if let Some(rank) = data_parallel_rank {
// Route to the engine at the specified rank index.
let engine_id = EngineId::from_engine_index(rank);
return self
.routing_per_engine
.contains_key(&engine_id)
.then_some(engine_id)
.ok_or_else(|| Error::InvalidDataParallelRank {
rank,
num_engines: self.routing_per_engine.len() as u32,
});
}
Ok(self
.routing_per_engine
.iter()
.min_by_key(|(_, state)| state.routing_score())
.map(|(engine_id, _)| engine_id.clone())
.expect("request registry must contain at least one engine"))
}
/// Filter the given request IDs to the subset that are still tracked as
/// active and can be aborted, grouped by engine.
pub fn abortable_request_ids(&self, request_ids: &[String]) -> BTreeMap<EngineId, Vec<String>> {
let mut by_engine = BTreeMap::new();
for request_id in request_ids {
let Some(tracked) = self.requests.get(request_id.as_str()) else {
continue;
};
by_engine
.entry(tracked.engine_id.clone())
.or_insert_with(Vec::new)
.push(request_id.clone());
}
by_engine
}
/// Obtain the stream sender for one output. If it indicates the request is
/// finished, it will be removed from the registry.
pub fn sender_for_output(&mut self, output: &EngineCoreOutput) -> Option<OutputSender> {
if output.finished() {
self.remove(output.request_id.as_str()).map(|tracked| tracked.0)
} else {
self.requests
.get(output.request_id.as_str())
.map(|tracked| tracked.sender.clone())
}
}
/// Remove a batch of requests that have finished or aborted, returning
/// their stream senders.
pub fn finish_many<'a>(
&mut self,
request_ids: impl IntoIterator<Item = &'a String>,
) -> Vec<OutputSender> {
request_ids
.into_iter()
.filter_map(|request_id| self.remove(request_id.as_str()).map(|tracked| tracked.0))
.collect()
}
/// Apply one scheduler stats update for the given engine to the local
/// routing state. Returns `false` if the engine is unknown to the
/// client.
pub fn apply_scheduler_stats(&mut self, engine_index: u32, stats: &SchedulerStats) -> bool {
self.apply_scheduler_counts(
engine_index,
EngineLoadSnapshot {
waiting: stats.num_waiting_reqs as usize,
running: stats.num_running_reqs as usize,
},
)
}
/// Mark the registry as closed, detach and return all tracked senders.
pub fn close(&mut self) -> Vec<OutputSender> {
if self.closed {
return Vec::new();
}
self.closed = true;
std::mem::take(&mut self.requests)
.into_values()
.map(|tracked| tracked.sender)
.collect()
}
/// Remove one request from the local registry. Returns the tracked entry if
/// it exists.
#[must_use]
pub fn remove(&mut self, request_id: &str) -> Option<(OutputSender, EngineId)> {
let tracked = self.requests.remove(request_id)?;
self.routing_per_engine
.get_mut(&tracked.engine_id)
.expect("request registry must track all known engines")
.inflight -= 1;
Some((tracked.sender, tracked.engine_id))
}
fn apply_scheduler_counts(&mut self, engine_index: u32, next: EngineLoadSnapshot) -> bool {
let engine_id = EngineId::from_engine_index(engine_index);
let Some(state) = self.routing_per_engine.get_mut(&engine_id) else {
return false;
};
let previous = state.last_scheduler_stats;
if previous != Some(next) {
trace!(
?engine_id,
previous_waiting = previous.map(|stats| stats.waiting),
previous_running = previous.map(|stats| stats.running),
waiting = next.waiting,
running = next.running,
"updated scheduler routing counts",
);
}
state.apply_scheduler_counts(next);
true
}
#[cfg(test)]
pub fn contains(&self, request_id: &str) -> bool {
self.requests.contains_key(request_id)
}
pub fn is_closed(&self) -> bool {
self.closed
}
}
/// Internal registry for tracking active utility calls and their waiting
/// receivers.
#[derive(Debug)]
pub struct UtilityRegistry {
closed: bool,
next_call_id: AtomicI64,
utility_calls: BTreeMap<i64, UtilitySender>,
}
impl Default for UtilityRegistry {
fn default() -> Self {
Self {
closed: false,
next_call_id: AtomicI64::new(1),
utility_calls: BTreeMap::default(),
}
}
}
impl UtilityRegistry {
/// Allocate the next utility `call_id` and register a newly added utility
/// call.
pub fn allocate_and_register(&mut self) -> (i64, UtilityReceiver) {
let call_id = self.next_call_id.fetch_add(1, Ordering::Relaxed);
let (tx, rx) = oneshot::channel();
self.utility_calls.insert(call_id, tx);
(call_id, rx)
}
/// Resolve a utility output to its waiting receiver.
pub fn resolve(&mut self, call_id: &i64) -> Option<UtilitySender> {
self.utility_calls.remove(call_id)
}
/// Drop a batch of registered utility calls without delivering a result.
/// Used to roll back allocations when the dispatch fan-out fails before
/// every engine could accept the request.
pub fn unregister_many(&mut self, call_ids: impl IntoIterator<Item = i64>) {
for call_id in call_ids {
self.utility_calls.remove(&call_id);
}
}
/// Mark the registry as closed, detach and return all tracked senders.
pub fn close(&mut self) -> Vec<UtilitySender> {
if self.closed {
return Vec::new();
}
self.closed = true;
std::mem::take(&mut self.utility_calls).into_values().collect()
}
#[cfg(test)]
pub fn contains(&self, call_id: i64) -> bool {
self.utility_calls.contains_key(&call_id)
}
pub fn is_closed(&self) -> bool {
self.closed
}
}
#[cfg(test)]
mod tests {
use super::{EngineRoutingState, RequestRegistry, UtilityRegistry};
use crate::EngineId;
use crate::client::state::EngineLoadSnapshot;
use crate::protocol::{EngineCoreFinishReason, EngineCoreOutput};
use crate::transport::ConnectedEngine;
#[test]
fn registry_rejects_duplicate_request_ids() {
let mut registry = RequestRegistry::new(&[ConnectedEngine {
engine_id: EngineId::from(b"engine-0"),
ready_response: None,
}]);
registry.register("req-1".to_string(), None).unwrap();
let error = registry.register("req-1".to_string(), None).unwrap_err();
assert!(matches!(
error,
crate::error::Error::DuplicateRequestId { request_id } if request_id == "req-1"
));
}
#[test]
fn registry_removes_finished_request_on_output() {
let mut registry = RequestRegistry::new(&[ConnectedEngine {
engine_id: EngineId::from(b"engine-0"),
ready_response: None,
}]);
registry.register("req-1".to_string(), None).unwrap();
let sender = registry.sender_for_output(&EngineCoreOutput {
request_id: "req-1".to_string(),
finish_reason: Some(EngineCoreFinishReason::Length),
..Default::default()
});
assert!(sender.is_some());
assert!(!registry.contains("req-1"));
}
#[test]
fn registry_closes_all_requests_on_failure() {
let mut registry = RequestRegistry::new(&[ConnectedEngine {
engine_id: EngineId::from(b"engine-0"),
ready_response: None,
}]);
registry.register("req-1".to_string(), None).unwrap();
registry.register("req-2".to_string(), None).unwrap();
let senders = registry.close();
assert_eq!(senders.len(), 2);
assert!(registry.is_closed());
}
#[test]
fn registry_tracks_engine_id_per_request() {
let engine_0 = EngineId::from_engine_index(0);
let engine_1 = EngineId::from_engine_index(1);
let mut registry = RequestRegistry::new(&[
ConnectedEngine {
engine_id: engine_0.clone(),
ready_response: None,
},
ConnectedEngine {
engine_id: engine_1.clone(),
ready_response: None,
},
]);
let (chosen_0, _) = registry.register("req-1".to_string(), None).unwrap();
let (chosen_1, _) = registry.register("req-2".to_string(), None).unwrap();
let (chosen_0_again, _) = registry.register("req-3".to_string(), None).unwrap();
assert_eq!(chosen_0, engine_0);
assert_eq!(chosen_1, engine_1);
assert_eq!(chosen_0_again, engine_0);
let grouped = registry.abortable_request_ids(&[
"req-1".to_string(),
"req-2".to_string(),
"req-3".to_string(),
]);
assert_eq!(
grouped.get(&engine_0).unwrap(),
&vec!["req-1".to_string(), "req-3".to_string()]
);
assert_eq!(grouped.get(&engine_1).unwrap(), &vec!["req-2".to_string()]);
}
#[test]
fn registry_uses_inflight_as_waiting_fallback_before_stats_arrive() {
let engine_0 = EngineId::from_engine_index(0);
let engine_1 = EngineId::from_engine_index(1);
let mut registry = RequestRegistry::new(&[
ConnectedEngine {
engine_id: engine_0.clone(),
ready_response: None,
},
ConnectedEngine {
engine_id: engine_1.clone(),
ready_response: None,
},
]);
let (chosen_0, _) = registry.register("req-1".to_string(), None).unwrap();
let (chosen_1, _) = registry.register("req-2".to_string(), None).unwrap();
let (chosen_0_again, _) = registry.register("req-3".to_string(), None).unwrap();
assert_eq!(chosen_0, engine_0);
assert_eq!(chosen_1, engine_1);
assert_eq!(chosen_0_again, engine_0);
}
#[test]
fn routing_score_uses_inflight_before_stats_arrive() {
let state = EngineRoutingState {
inflight: 3,
last_scheduler_stats: None,
};
assert_eq!(state.routing_score(), 3);
}
#[test]
fn routing_score_uses_inflight_as_scheduler_stats_lower_bound() {
let state = EngineRoutingState {
inflight: 7,
last_scheduler_stats: Some(EngineLoadSnapshot {
waiting: 0,
running: 2,
}),
};
assert_eq!(state.routing_score(), 7);
}
#[test]
fn routing_score_keeps_extra_waiting_penalty() {
let state = EngineRoutingState {
inflight: 1,
last_scheduler_stats: Some(EngineLoadSnapshot {
waiting: 3,
running: 2,
}),
};
assert_eq!(state.routing_score(), 14);
}
#[test]
fn registry_prefers_real_scheduler_stats_over_inflight() {
let engine_0 = EngineId::from_engine_index(0);
let engine_1 = EngineId::from_engine_index(1);
let mut registry = RequestRegistry::new(&[
ConnectedEngine {
engine_id: engine_0.clone(),
ready_response: None,
},
ConnectedEngine {
engine_id: engine_1.clone(),
ready_response: None,
},
]);
assert!(registry.apply_scheduler_counts(
0,
EngineLoadSnapshot {
waiting: 3,
running: 2
}
));
assert!(registry.apply_scheduler_counts(
1,
EngineLoadSnapshot {
waiting: 0,
running: 1
}
));
let (chosen, _) = registry.register("req-stats".to_string(), None).unwrap();
assert_eq!(chosen, engine_1);
}
#[test]
fn register_with_data_parallel_rank_routes_to_specified_engine() {
let engine_0 = EngineId::from_engine_index(0);
let engine_1 = EngineId::from_engine_index(1);
let engine_2 = EngineId::from_engine_index(2);
let mut registry = RequestRegistry::new(&[
ConnectedEngine {
engine_id: engine_0.clone(),
ready_response: None,
},
ConnectedEngine {
engine_id: engine_1.clone(),
ready_response: None,
},
ConnectedEngine {
engine_id: engine_2.clone(),
ready_response: None,
},
]);
// Explicitly target rank 2 (third engine).
let (chosen, _) = registry.register("req-1".to_string(), Some(2)).unwrap();
assert_eq!(chosen, engine_2);
// Explicitly target rank 0 (first engine).
let (chosen, _) = registry.register("req-2".to_string(), Some(0)).unwrap();
assert_eq!(chosen, engine_0);
// Explicitly target rank 1.
let (chosen, _) = registry.register("req-3".to_string(), Some(1)).unwrap();
assert_eq!(chosen, engine_1);
}
#[test]
fn register_with_data_parallel_rank_bypasses_load_balancing() {
let engine_0 = EngineId::from_engine_index(0);
let engine_1 = EngineId::from_engine_index(1);
let mut registry = RequestRegistry::new(&[
ConnectedEngine {
engine_id: engine_0.clone(),
ready_response: None,
},
ConnectedEngine {
engine_id: engine_1.clone(),
ready_response: None,
},
]);
// Load-balance: first two go to engine_0 and engine_1.
registry.register("req-lb-0".to_string(), None).unwrap();
// Now engine_0 has 1 in-flight. Without dp_rank, next would go to engine_1.
// But with dp_rank=0, it should still go to engine_0.
let (chosen, _) = registry.register("req-dp".to_string(), Some(0)).unwrap();
assert_eq!(chosen, engine_0);
}
#[test]
fn register_with_out_of_range_rank_returns_error() {
let mut registry = RequestRegistry::new(&[
ConnectedEngine {
engine_id: EngineId::from_engine_index(0),
ready_response: None,
},
ConnectedEngine {
engine_id: EngineId::from_engine_index(1),
ready_response: None,
},
]);
let error = registry.register("req-1".to_string(), Some(2)).unwrap_err();
assert!(matches!(
error,
crate::error::Error::InvalidDataParallelRank {
rank: 2,
num_engines: 2,
}
));
}
#[test]
fn register_with_rank_on_single_engine_only_accepts_zero() {
let engine_0 = EngineId::from_engine_index(0);
let mut registry = RequestRegistry::new(&[ConnectedEngine {
engine_id: engine_0.clone(),
ready_response: None,
}]);
let (chosen, _) = registry.register("req-ok".to_string(), Some(0)).unwrap();
assert_eq!(chosen, engine_0);
let error = registry.register("req-bad".to_string(), Some(1)).unwrap_err();
assert!(matches!(
error,
crate::error::Error::InvalidDataParallelRank {
rank: 1,
num_engines: 1,
}
));
}
#[test]
fn utility_registry_tracks_and_removes_call_ids() {
let mut registry = UtilityRegistry::default();
let (call_id_1, _) = registry.allocate_and_register();
let (call_id_2, _) = registry.allocate_and_register();
assert_eq!(call_id_1, 1);
assert_eq!(call_id_2, 2);
assert!(registry.contains(1));
assert!(registry.contains(2));
assert!(registry.resolve(&1).is_some());
assert!(!registry.contains(1));
assert!(registry.contains(2));
}
#[test]
fn utility_registry_closes_all_waiters_on_failure() {
let mut registry = UtilityRegistry::default();
registry.allocate_and_register();
registry.allocate_and_register();
let senders = registry.close();
assert_eq!(senders.len(), 2);
assert!(!registry.contains(1));
assert!(!registry.contains(2));
assert!(registry.is_closed());
}
#[test]
fn utility_registry_unregister_many_drops_pending_calls() {
use tokio::sync::oneshot::error::TryRecvError;
let mut registry = UtilityRegistry::default();
let (call_id_1, mut rx_1) = registry.allocate_and_register();
let (call_id_2, mut rx_2) = registry.allocate_and_register();
let (call_id_3, _rx_3) = registry.allocate_and_register();
// Drop two of the three allocated calls; the third stays pending.
registry.unregister_many([call_id_1, call_id_2]);
assert!(!registry.contains(call_id_1));
assert!(!registry.contains(call_id_2));
assert!(registry.contains(call_id_3));
// The receivers must observe the sender being dropped (channel closed).
assert!(matches!(rx_1.try_recv(), Err(TryRecvError::Closed)));
assert!(matches!(rx_2.try_recv(), Err(TryRecvError::Closed)));
}
#[test]
fn utility_registry_unregister_many_ignores_unknown_call_ids() {
let mut registry = UtilityRegistry::default();
let (call_id, _rx) = registry.allocate_and_register();
// Unknown call ids are silently ignored — caller doesn't care which were live.
registry.unregister_many([call_id, 42, 9999]);
assert!(!registry.contains(call_id));
}
}
@@ -0,0 +1,153 @@
use std::ops::Deref;
use std::pin::Pin;
use std::task::{Context, Poll};
use futures::Stream;
use futures::stream::FusedStream;
use thiserror_ext::AsReport as _;
use tokio::sync::mpsc;
use tracing::{debug, error, warn};
use crate::client::AbortRequest;
use crate::client::state::OutputReceiver;
use crate::protocol::{EngineCoreFinishReason, EngineCoreOutput};
use crate::{AbortCause, Error, Result};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum State {
Running,
Finished,
ClosedWithError,
UnexpectedClose,
}
/// One request-scoped engine-core output plus the enclosing batch metadata.
#[derive(Debug, Clone, PartialEq)]
pub struct EngineCoreStreamOutput {
pub engine_index: u32,
pub timestamp: f64,
pub output: EngineCoreOutput,
}
impl Deref for EngineCoreStreamOutput {
type Target = EngineCoreOutput;
fn deref(&self) -> &Self::Target {
&self.output
}
}
/// Stream of raw engine-core outputs for one request.
///
/// The stream yields only [`EngineCoreStreamOutput`] values whose embedded
/// output `request_id` matches the originating `add_request()` call. Normal
/// request completion is expected to include a final output object whose
/// `finish_reason` is non-`None`.
pub struct EngineCoreOutputStream {
request_id: String,
abort_tx: mpsc::UnboundedSender<AbortRequest>,
state: State,
rx: OutputReceiver,
}
impl EngineCoreOutputStream {
pub(crate) fn new(
request_id: String,
abort_tx: mpsc::UnboundedSender<AbortRequest>,
rx: OutputReceiver,
) -> Self {
Self {
request_id,
abort_tx,
state: State::Running,
rx,
}
}
/// Return the engine-core `request_id` bound to this stream.
pub fn request_id(&self) -> &str {
&self.request_id
}
}
impl Stream for EngineCoreOutputStream {
type Item = Result<EngineCoreStreamOutput>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
if self.is_terminated() {
return Poll::Ready(None);
}
match Pin::new(&mut self.rx).poll_recv(cx) {
Poll::Pending => Poll::Pending,
Poll::Ready(Some(item)) => {
match &item {
Ok(output) => {
// If the output indicates the request is finished, mark the stream as
// terminated with cleanly-finished state and expect no more outputs to
// come.
if output.finished() {
if output.finish_reason == Some(EngineCoreFinishReason::Error) {
error!(
self.request_id,
"request failed with an internal error during generation"
);
}
debug!(self.request_id, "request completed via final output");
self.state = State::Finished;
}
}
Err(error) => {
// If we get an error from the output stream, mark the stream as terminated
// with an error.
warn!(self.request_id, error = %error.as_report(), "request encountered an error");
self.state = State::ClosedWithError;
}
}
Poll::Ready(Some(item))
}
Poll::Ready(None) => {
// If we get a `None` without seeing a finished output, this is an unexpected
// close from the engine side. Mark the stream as terminated
// with an unexpected close state and send an error down the
// stream to notify the caller.
warn!(self.request_id, "request stream closed unexpectedly");
self.state = State::UnexpectedClose;
Poll::Ready(Some(Err(Error::RequestStreamClosed {
request_id: self.request_id.clone(),
})))
}
}
}
}
impl FusedStream for EngineCoreOutputStream {
fn is_terminated(&self) -> bool {
!matches!(self.state, State::Running)
}
}
impl Drop for EngineCoreOutputStream {
fn drop(&mut self) {
if self.is_terminated() {
// If it's terminated, it means that the request either finished cleanly, or
// encountered an error or unexpected close from the engine. In any
// case, the request stream is already considered inactive and
// there's no need to abort it on the engine side.
return;
}
let abort_req = AbortRequest {
request_id: self.request_id.clone(),
cause: AbortCause::current(),
};
if self.abort_tx.send(abort_req).is_err() {
warn!(
request_id = self.request_id,
"auto-abort worker already shut down; skip auto-abort"
);
}
}
}
@@ -0,0 +1,91 @@
use std::time::Duration;
use bytes::Bytes;
use zeromq::prelude::{Socket, SocketRecv, SocketSend};
use zeromq::{PullSocket, XPubSocket, ZmqMessage};
use crate::error::{Error, Result, bail_unexpected_handshake_message};
/// Engine-facing sockets owned by the in-process coordinator.
pub(crate) struct CoordinatorBootstrap {
pub input_address: String,
pub output_address: String,
pub input_socket: XPubSocket,
pub output_socket: PullSocket,
}
impl CoordinatorBootstrap {
/// Bind the engine-facing coordinator sockets on the given host.
pub(crate) async fn bind(local_host: &str) -> Result<Self> {
let mut input_socket = XPubSocket::new();
let input_address = input_socket.bind(&format!("tcp://{local_host}:0")).await?.to_string();
let mut output_socket = PullSocket::new();
let output_address =
output_socket.bind(&format!("tcp://{local_host}:0")).await?.to_string();
Ok(Self {
input_address,
output_address,
input_socket,
output_socket,
})
}
/// Complete the engine-facing startup gate before engines are allowed to
/// send handshake READY.
pub(crate) async fn wait_for_startup_gate(
&mut self,
engine_count: usize,
ready_timeout: Duration,
) -> Result<()> {
wait_for_engine_subscriptions(&mut self.input_socket, engine_count, ready_timeout).await?;
send_ready_to_engines(&mut self.input_socket).await?;
Ok(())
}
}
/// Wait until all engines subscribe to the coordinator broadcast socket.
async fn wait_for_engine_subscriptions(
input_socket: &mut XPubSocket,
engine_count: usize,
ready_timeout: Duration,
) -> Result<()> {
let mut received = 0;
while received < engine_count {
let message =
tokio::time::timeout(ready_timeout, input_socket.recv()).await.map_err(|_| {
Error::HandshakeTimeout {
stage: "coordinator engine subscriptions",
timeout: ready_timeout,
}
})??;
if message.len() != 1 {
bail_unexpected_handshake_message!(
"expected 1 frame for coordinator subscription, got {}",
message.len()
);
}
let frame = message
.into_vec()
.into_iter()
.next()
.expect("single-frame coordinator subscription message");
if frame.as_ref() != [0x01] {
bail_unexpected_handshake_message!(
"expected coordinator subscription frame [0x01], got {:?}",
frame.as_ref()
);
}
received += 1;
}
Ok(())
}
/// Send the coordinator READY marker to all subscribed engines.
async fn send_ready_to_engines(input_socket: &mut XPubSocket) -> Result<()> {
input_socket.send(ZmqMessage::from(Bytes::from_static(b"READY"))).await?;
Ok(())
}
@@ -0,0 +1,167 @@
use std::sync::Arc;
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
use thiserror_ext::AsReport;
use tokio::sync::mpsc;
use tracing::{debug, warn};
use zeromq::prelude::{SocketRecv, SocketSend};
use zeromq::{XSubSocket, ZmqMessage};
use crate::client::imp::ClientInner;
use crate::coordinator::handle::{CoordinatorCommand, CoordinatorState};
use crate::error::{Error, Result, bail_unexpected_coordinator_output};
use crate::protocol::{OpaqueValue, decode_msgpack, encode_msgpack};
/// Frontend-to-coordinator wakeup message sent when the first request arrives
/// while all engines are paused.
///
/// This matches the frontend-side msgpack tuple sent by Python
/// `DPAsyncMPClient._ensure_stats_update_task` to the coordinator front socket.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/694449050f8dac3d9853e97e518b4a43ec52106a/vllm/v1/engine/core_client.py#L1230-L1236>
#[derive(Debug, Clone, PartialEq, Eq, Serialize_tuple, Deserialize_tuple)]
struct CoordinatorWakeupMessage {
/// Engine index that already has the triggering request and should be
/// excluded from the coordinator's `START_DP_WAVE` rebroadcast.
exclude_engine_index: u32,
/// DP wave number observed by the frontend when the request was admitted.
wave: u32,
}
/// Coordinator-to-frontend state publish received on the front-side coordinator
/// socket.
///
/// This matches the msgpack tuple periodically published by Python
/// `DPCoordinatorProc.run_coordinator` to all connected frontends.
///
/// Original Python definitions:
/// <https://github.com/vllm-project/vllm/blob/694449050f8dac3d9853e97e518b4a43ec52106a/vllm/v1/engine/coordinator.py#L282-L283>
/// <https://github.com/vllm-project/vllm/blob/694449050f8dac3d9853e97e518b4a43ec52106a/vllm/v1/engine/coordinator.py#L445-L447>
#[derive(Debug, Clone, PartialEq, Deserialize_tuple)]
struct CoordinatorStateUpdate {
/// Global per-engine request counts published by the coordinator.
///
/// The Rust bootstrapped external-coordinator path preserves this field for
/// wire compatibility but intentionally ignores it for routing decisions.
counts: OpaqueValue,
/// Current global DP wave number stamped onto newly admitted requests.
wave: u32,
/// Whether engines are currently running (`true`) or paused (`false`).
engines_running: bool,
}
/// Background half of an external Python-owned coordinator connection.
///
/// This owns the command receiver and one frontend-facing XSUB socket. It
/// mirrors the subset of Python's coordinator protocol needed by the Rust
/// bootstrapped frontend: receive `(counts, wave, running)` publishes, ignore
/// `counts`, and send `(exclude_engine_index, wave)` wakeup messages when the
/// first request arrives while engines are paused.
pub(crate) struct ExternalCoordinatorService {
state: Arc<CoordinatorState>,
command_rx: mpsc::UnboundedReceiver<CoordinatorCommand>,
socket: XSubSocket,
}
impl ExternalCoordinatorService {
pub(super) fn new(
state: Arc<CoordinatorState>,
command_rx: mpsc::UnboundedReceiver<CoordinatorCommand>,
socket: XSubSocket,
) -> Self {
Self {
state,
command_rx,
socket,
}
}
/// Apply one frontend-originated command to the external coordinator state
/// machine.
async fn handle_command(&mut self, command: CoordinatorCommand) -> Result<()> {
match command {
CoordinatorCommand::FirstRequest {
target_engine_id,
wave,
} => {
let target_engine_index = target_engine_id.engine_index().ok_or_else(|| {
Error::UnsupportedCoordinatorEngineId {
engine_id: target_engine_id.to_vec(),
}
})?;
debug!(
wave,
exclude_engine_index = target_engine_index,
"notifying external coordinator about first request while engines were paused"
);
let payload = encode_msgpack(&CoordinatorWakeupMessage {
exclude_engine_index: target_engine_index,
wave,
})?;
self.socket.send(ZmqMessage::from(payload)).await?;
}
}
Ok(())
}
/// Apply one publish received from the xsub socket containing a coordinator
/// state update.
async fn handle_publish(&mut self, message: ZmqMessage) -> Result<()> {
let frames = message.into_vec();
if frames.len() != 1 {
bail_unexpected_coordinator_output!(
"received malformed external coordinator publish with {} frame(s)",
frames.len()
);
}
let update: CoordinatorStateUpdate = decode_msgpack(&frames[0])?;
let mut state = self.state.lock();
let previous_wave = state.current_wave;
let previous_engines_running = state.engines_running;
state.current_wave = update.wave;
state.engines_running = update.engines_running;
debug!(
previous_wave,
wave = update.wave,
previous_engines_running,
engines_running = update.engines_running,
"applied external coordinator state update"
);
Ok(())
}
/// Drive the coordinator event loop until either side of the control plane
/// is closed or a fatal error is observed.
pub(crate) async fn run(mut self, inner: Arc<ClientInner>) {
let result: Result<()> = async {
loop {
tokio::select! {
// Received frontend-originated command from the handle.
command = self.command_rx.recv() => {
let Some(command) = command else {
warn!("external coordinator command channel closed, shutting down service");
return Ok(());
};
self.handle_command(command).await?;
}
// Received publish from the external coordinator socket.
publish = self.socket.recv() => {
let publish = publish.map_err(Error::from)?;
self.handle_publish(publish).await?;
}
}
}
}
.await;
let Err(error) = result else { return };
warn!(
error = %error.as_report(),
"external coordinator service exiting with error"
);
inner.close_registries(Arc::new(error));
}
}
@@ -0,0 +1,123 @@
use std::sync::Arc;
use parking_lot::Mutex;
use tokio::sync::mpsc;
use zeromq::prelude::Socket;
use zeromq::{XPubSocket, XSubSocket};
use crate::coordinator::external::ExternalCoordinatorService;
use crate::coordinator::inproc::InProcCoordinatorRunner;
use crate::error::{Error, Result, bail_control_closed};
use crate::transport::EngineId;
/// Snapshot to the coordinator state for request routing and stamping.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct CoordinatorStateSnapshot {
/// The current DP wave, which will be stamped on outgoing requests.
pub current_wave: u32,
/// Whether the engines are currently running or paused, which determines if
/// the frontend must trigger a new wave on the next request.
pub engines_running: bool,
}
/// Shared in-process coordinator state.
pub(crate) type CoordinatorState = Mutex<CoordinatorStateSnapshot>;
/// Commands sent from the frontend request path into the background runner.
#[derive(Debug)]
pub(crate) enum CoordinatorCommand {
/// The first request arrived while all engines were paused.
///
/// The coordinator should broadcast `START_DP_WAVE` with the current wave
/// and the target engine index as the excluded engine.
FirstRequest {
target_engine_id: EngineId,
wave: u32,
},
}
/// Frontend-facing coordinator handle used by `EngineCoreClient::call()`.
///
/// This side stays intentionally small: it can read the latest wave snapshot
/// and enqueue a `FirstRequest` transition when the request path observes the
/// system in the paused state.
#[derive(Clone)]
pub(crate) struct CoordinatorHandle {
state: Arc<CoordinatorState>,
command_tx: mpsc::UnboundedSender<CoordinatorCommand>,
}
impl CoordinatorHandle {
fn new_parts() -> (
Self,
Arc<CoordinatorState>,
mpsc::UnboundedReceiver<CoordinatorCommand>,
) {
let state = Arc::new(Mutex::new(CoordinatorStateSnapshot {
current_wave: 0,
engines_running: false,
}));
let (command_tx, command_rx) = mpsc::unbounded_channel();
(
Self {
state: state.clone(),
command_tx,
},
state,
command_rx,
)
}
/// Build the paired frontend handle and background runner around one
/// engine-facing coordinator broadcast socket.
pub(crate) fn new_inproc(coordinator_input: XPubSocket) -> (Self, InProcCoordinatorRunner) {
let (handle, state, command_rx) = Self::new_parts();
(
handle,
InProcCoordinatorRunner::new(state, command_rx, coordinator_input),
)
}
/// Build the paired frontend handle and background service around an
/// external Python-owned frontend-side coordinator socket.
pub(crate) async fn connect_external(
coordinator_address: &str,
) -> Result<(Self, ExternalCoordinatorService)> {
let (handle, state, command_rx) = Self::new_parts();
let mut socket = XSubSocket::new();
socket.connect(coordinator_address).await?;
socket.subscribe("").await?;
Ok((
handle,
ExternalCoordinatorService::new(state, command_rx, socket),
))
}
/// Snapshot the coordinator state for request routing and stamping.
pub(crate) fn snapshot(&self) -> CoordinatorStateSnapshot {
*self.state.lock()
}
/// Notify the runner that a new request arrived while engines were paused.
///
/// The handle flips `engines_running` optimistically so concurrent request
/// submissions coalesce behind one `START_DP_WAVE` broadcast instead of all
/// trying to trigger the wave independently.
pub(crate) fn notify_first_request(&self, target_engine_id: EngineId) -> Result<()> {
let mut state = self.state.lock();
if state.engines_running {
return Ok(());
}
let command = CoordinatorCommand::FirstRequest {
target_engine_id,
wave: state.current_wave,
};
if self.command_tx.send(command).is_err() {
bail_control_closed!("in-process coordinator command channel already shut down");
}
state.engines_running = true;
Ok(())
}
}
@@ -0,0 +1,204 @@
use std::sync::Arc;
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
use thiserror_ext::AsReport;
use tokio::sync::mpsc;
use tracing::{debug, warn};
use zeromq::prelude::SocketSend;
use zeromq::{XPubSocket, ZmqMessage};
use crate::client::imp::ClientInner;
use crate::coordinator::handle::{CoordinatorCommand, CoordinatorState};
use crate::error::{Error, Result, bail_unexpected_coordinator_output};
use crate::protocol::{
ClassifiedEngineCoreOutputs, DpControlMessage, EngineCoreOutputs, EngineCoreRequestType,
encode_msgpack,
};
/// Coordinator-to-engine `START_DP_WAVE` control payload encoded on the
/// engine-facing coordinator socket.
///
/// This matches the msgpack tuple broadcast by Python
/// `DPCoordinatorProc._send_start_wave`.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/694449050f8dac3d9853e97e518b4a43ec52106a/vllm/v1/engine/coordinator.py#L453-L459>
#[derive(Debug, Clone, PartialEq, Eq, Serialize_tuple, Deserialize_tuple)]
struct StartDpWaveMessage {
/// DP wave number that all engines should start processing.
wave: u32,
/// Engine index that already received the triggering request and should not
/// receive an extra wakeup notification.
exclude_engine_index: u32,
}
/// Background half of the in-process coordinator.
///
/// This owns the command receiver and the engine-facing coordinator input
/// socket. It is the single place where wave transitions are serialized and
/// where `START_DP_WAVE` broadcasts are emitted.
pub(crate) struct InProcCoordinatorRunner {
state: Arc<CoordinatorState>,
command_rx: mpsc::UnboundedReceiver<CoordinatorCommand>,
coordinator_input: XPubSocket,
}
impl InProcCoordinatorRunner {
pub(super) fn new(
state: Arc<CoordinatorState>,
command_rx: mpsc::UnboundedReceiver<CoordinatorCommand>,
coordinator_input: XPubSocket,
) -> Self {
Self {
state,
command_rx,
coordinator_input,
}
}
/// Broadcast Python-compatible `START_DP_WAVE` to all connected engines.
async fn broadcast_start_wave(&mut self, wave: u32, exclude_engine_index: u32) -> Result<()> {
let payload = encode_msgpack(&StartDpWaveMessage {
wave,
exclude_engine_index,
})?;
self.coordinator_input
.send(
ZmqMessage::try_from(vec![
EngineCoreRequestType::StartDpWave.to_frame(),
payload.into(),
])
.expect("coordinator START_DP_WAVE message must contain two frames"),
)
.await?;
Ok(())
}
/// Apply one frontend-originated command to the coordinator state machine.
async fn handle_command(&mut self, command: CoordinatorCommand) -> Result<()> {
match command {
CoordinatorCommand::FirstRequest {
target_engine_id,
wave,
} => {
let target_engine_index = target_engine_id.engine_index().ok_or_else(|| {
Error::UnsupportedCoordinatorEngineId {
engine_id: target_engine_id.to_vec(),
}
})?;
self.state.lock().current_wave = wave;
debug!(
wave,
exclude_engine_index = target_engine_index,
"starting DP wave after first request while engines were paused"
);
self.broadcast_start_wave(wave, target_engine_index).await?;
}
}
Ok(())
}
/// Apply one engine-originated control output to the coordinator state
/// machine.
async fn handle_outputs(&mut self, outputs: EngineCoreOutputs) -> Result<()> {
match outputs.classify() {
ClassifiedEngineCoreOutputs::RequestBatch(batch)
if batch.outputs.is_empty() && batch.finished_requests.is_none() =>
{
// Stats-only output for coordinator.
// Ignore since the Rust coordinator doesn't track stats for
// routing decisions.
}
ClassifiedEngineCoreOutputs::DpControl {
engine_index,
control,
..
} => match control {
// The engines signals they completed the current wave and are now paused.
// Advance the current wave and mark the state as paused.
DpControlMessage::WaveComplete(wave) => {
let mut state = self.state.lock();
if wave >= state.current_wave {
let next_wave = wave + 1;
debug!(
wave,
next_wave,
"DP wave finished; pausing engines and advancing coordinator state"
);
state.current_wave = wave + 1;
state.engines_running = false;
}
}
// An engine requests to start the wave.
// Rebroadcast the wave to all engines except for the originated one.
DpControlMessage::StartWave(wave) => {
let should_broadcast = {
let mut state = self.state.lock();
if wave > state.current_wave
|| (wave == state.current_wave && !state.engines_running)
{
state.current_wave = wave;
state.engines_running = true;
true
} else {
false
}
};
if should_broadcast {
debug!(
wave,
exclude_engine_index = engine_index,
"starting DP wave after stale-wave notification from engine"
);
self.broadcast_start_wave(wave, engine_index).await?;
}
}
},
other => {
bail_unexpected_coordinator_output!(
"received non-control output on coordinator path: {other:?}"
);
}
}
Ok(())
}
/// Drive the coordinator event loop until either side of the control plane
/// is closed or a fatal error is observed.
///
/// Any fatal error closes the main client registries so request streams and
/// future calls observe a stable shutdown cause.
pub(crate) async fn run(
mut self,
mut output_rx: mpsc::Receiver<Result<EngineCoreOutputs>>,
inner: Arc<ClientInner>,
) {
let result: Result<()> = async {
loop {
tokio::select! {
// Received frontend-originated command from the handle.
command = self.command_rx.recv() => {
let Some(command) = command else {
warn!("coordinator command channel closed, shutting down coordinator runner");
return Ok(());
};
self.handle_command(command).await?;
}
// Received engine-originated control output from the coordinator socket.
outputs = output_rx.recv() => {
let Some(outputs) = outputs else {
warn!("coordinator output channel closed, shutting down coordinator runner");
return Ok(());
};
self.handle_outputs(outputs?).await?;
}
}
}
}
.await;
let Err(error) = result else { return };
warn!(error = %error.as_report(), "coordinator runner exiting with error");
inner.close_registries(Arc::new(error));
}
}
@@ -0,0 +1,7 @@
mod bootstrap;
mod external;
mod handle;
mod inproc;
pub(crate) use bootstrap::CoordinatorBootstrap;
pub(crate) use handle::CoordinatorHandle;
+88
View File
@@ -0,0 +1,88 @@
use std::sync::Arc;
use std::time::Duration;
use thiserror::Error;
use thiserror_ext::Macro;
pub type Result<T> = std::result::Result<T, Error>;
/// Public error type for the Rust engine-core client.
#[derive(Debug, Error, Macro)]
pub enum Error {
#[error("messagepack encode failed for {target_type}: {message}")]
Encode {
target_type: &'static str,
message: String,
},
#[error("messagepack decode failed for {target_type}: {message}")]
Decode {
target_type: &'static str,
message: String,
},
#[error("messagepack value decode failed")]
ValueDecode(#[from] rmpv::decode::Error),
#[error("messagepack ext value decode failed: {message}")]
ExtValueDecode { message: String },
#[error("io error")]
Io(#[from] std::io::Error),
#[error("transport error")]
Transport(#[from] zeromq::ZmqError),
#[error("engine core reported fatal failure")]
EngineCoreDead,
#[error("startup handshake timed out while waiting for {stage} after {timeout:?}")]
HandshakeTimeout {
stage: &'static str,
timeout: Duration,
},
#[error("engine input registration timed out after {timeout:?}")]
InputRegistrationTimeout { timeout: Duration },
#[error("unexpected engine id in startup handshake: expected {expected:?}, got {actual:?}")]
UnexpectedHandshakeIdentity { expected: Vec<u8>, actual: Vec<u8> },
#[error("unexpected startup handshake message: {message}")]
UnexpectedHandshakeMessage { message: String },
#[error("unexpected non-control output on coordinator path: {message}")]
UnexpectedCoordinatorOutput { message: String },
#[error("unexpected output on main dispatcher path: {message}")]
UnexpectedDispatcherOutput { message: String },
#[error("coordinator requires a Python-compatible two-byte engine id, got {engine_id:?}")]
UnsupportedCoordinatorEngineId { engine_id: Vec<u8> },
#[error("unsupported auxiliary frame(s): expected 1 frame, got {frame_count}")]
UnsupportedAuxFrames { frame_count: usize },
#[error("external coordinator mode is not implemented yet")]
UnsupportedExternalCoordinator,
#[error("unsupported field `{field}` in {context}")]
UnsupportedField {
context: &'static str,
field: &'static str,
},
#[error("engine control channel closed unexpectedly: {message}")]
ControlClosed { message: String },
#[error("request `{request_id}` is already in flight")]
DuplicateRequestId { request_id: String },
#[error("data parallel rank {rank} is out of range for {num_engines} engine(s)")]
InvalidDataParallelRank { rank: u32, num_engines: u32 },
#[error("engine-core output dispatcher closed: {message}")]
DispatcherClosed { message: String },
#[error("engine-core client is closed: {message}")]
ClientClosed { message: String },
#[error("request output stream for `{request_id}` closed unexpectedly")]
RequestStreamClosed { request_id: String },
#[error("utility call `{method}` failed (call_id={call_id}): {message}")]
UtilityCallFailed {
method: String,
call_id: i64,
message: String,
},
#[error("utility call `{method}` returned an invalid result (call_id={call_id}): {message}")]
UtilityResultDecode {
method: String,
call_id: i64,
message: String,
},
#[error("utility call `{method}` closed unexpectedly (call_id={call_id})")]
UtilityCallClosed { method: String, call_id: i64 },
/// A special variant to allow cloning the same error.
#[error(transparent)]
Shared(Arc<Self>),
}
+18
View File
@@ -0,0 +1,18 @@
mod client;
mod coordinator;
mod error;
mod metrics;
pub mod protocol;
#[cfg(any(test, feature = "test-util"))]
pub mod test_utils;
mod transport;
pub use client::{
AbortCause, CoordinatorMode, EngineCoreClient, EngineCoreClientConfig, EngineCoreOutputStream,
EngineCoreStreamOutput, TransportMode,
};
pub use error::{Error, Result};
pub use transport::{ENGINE_CORE_DEAD_SENTINEL, EngineId};
#[cfg(test)]
mod tests;
+131
View File
@@ -0,0 +1,131 @@
use vllm_metrics::{EngineLabels, EnginePositionLabels, SchedulerMetrics, WaitingReasonLabels};
use crate::protocol::stats::SchedulerStats;
const WAITING_REASON_CAPACITY: &str = "capacity";
const WAITING_REASON_DEFERRED: &str = "deferred";
/// Record the scheduler-stats-backed metrics for one engine at one point in
/// time.
pub(crate) fn record_scheduler_stats(
metrics: &SchedulerMetrics,
model_name: impl Into<String>,
engine: u32,
stats: &SchedulerStats,
) {
let model_name = model_name.into();
let labels = EngineLabels {
model_name: model_name.clone(),
engine,
};
// Scheduler state gauges.
metrics.scheduler_running.get_or_create(&labels).set(stats.num_running_reqs);
metrics
.scheduler_waiting
.get_or_create(&labels)
.set(stats.num_waiting_reqs + stats.num_skipped_waiting_reqs);
metrics
.scheduler_waiting_by_reason
.get_or_create(&WaitingReasonLabels {
model_name: model_name.clone(),
engine,
reason: WAITING_REASON_CAPACITY,
})
.set(stats.num_waiting_reqs);
metrics
.scheduler_waiting_by_reason
.get_or_create(&WaitingReasonLabels {
model_name: model_name.clone(),
engine,
reason: WAITING_REASON_DEFERRED,
})
.set(stats.num_skipped_waiting_reqs);
metrics.kv_cache_usage.get_or_create(&labels).set(stats.kv_cache_usage);
// Prefix-cache counters, including the connector-backed external cache path.
metrics
.prefix_cache_queries
.get_or_create(&labels)
.inc_by(stats.prefix_cache_stats.base.queries);
metrics
.prefix_cache_hits
.get_or_create(&labels)
.inc_by(stats.prefix_cache_stats.base.hits);
if let Some(connector_prefix_cache_stats) = &stats.connector_prefix_cache_stats {
metrics
.external_prefix_cache_queries
.get_or_create(&labels)
.inc_by(connector_prefix_cache_stats.base.queries);
metrics
.external_prefix_cache_hits
.get_or_create(&labels)
.inc_by(connector_prefix_cache_stats.base.hits);
}
// Speculative decoding counters.
if let Some(spec_decoding_stats) = &stats.spec_decoding_stats {
metrics
.spec_decode_num_drafts
.get_or_create(&labels)
.inc_by(spec_decoding_stats.num_drafts);
metrics
.spec_decode_num_draft_tokens
.get_or_create(&labels)
.inc_by(spec_decoding_stats.num_draft_tokens);
metrics
.spec_decode_num_accepted_tokens
.get_or_create(&labels)
.inc_by(spec_decoding_stats.num_accepted_tokens);
for (position, accepted_tokens) in
spec_decoding_stats.num_accepted_tokens_per_pos.iter().copied().enumerate()
{
metrics
.spec_decode_num_accepted_tokens_per_pos
.get_or_create(&EnginePositionLabels {
model_name: model_name.clone(),
engine,
position: position as u32,
})
.inc_by(accepted_tokens);
}
}
// Per-engine performance / MFU counters.
if let Some(perf_stats) = &stats.perf_stats
&& (perf_stats.num_flops_per_gpu != 0
|| perf_stats.num_read_bytes_per_gpu != 0
|| perf_stats.num_write_bytes_per_gpu != 0)
{
metrics
.estimated_flops_per_gpu
.get_or_create(&labels)
.inc_by(perf_stats.num_flops_per_gpu);
metrics
.estimated_read_bytes_per_gpu
.get_or_create(&labels)
.inc_by(perf_stats.num_read_bytes_per_gpu);
metrics
.estimated_write_bytes_per_gpu
.get_or_create(&labels)
.inc_by(perf_stats.num_write_bytes_per_gpu);
}
// Sampled KV-cache residency histograms.
if !stats.kv_cache_eviction_events.is_empty() {
let kv_block_lifetime_seconds = metrics.kv_block_lifetime_seconds.get_or_create(&labels);
let kv_block_idle_before_evict_seconds =
metrics.kv_block_idle_before_evict_seconds.get_or_create(&labels);
let kv_block_reuse_gap_seconds = metrics.kv_block_reuse_gap_seconds.get_or_create(&labels);
for event in &stats.kv_cache_eviction_events {
kv_block_lifetime_seconds.observe(event.lifetime_seconds);
kv_block_idle_before_evict_seconds.observe(event.idle_seconds);
for reuse_gap_seconds in &event.reuse_gaps_seconds {
kv_block_reuse_gap_seconds.observe(*reuse_gap_seconds);
}
}
}
}
@@ -0,0 +1,251 @@
use std::collections::BTreeSet;
use enum_as_inner::EnumAsInner;
use super::{EngineCoreOutput, EngineCoreOutputs, UtilityOutput};
use crate::protocol::stats::SchedulerStats;
/// Data-parallel control notifications multiplexed through `EngineCoreOutputs`.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum DpControlMessage {
WaveComplete(u32),
StartWave(u32),
}
#[derive(Debug, Clone, PartialEq)]
pub struct RequestBatchOutputs {
pub engine_index: u32,
pub outputs: Vec<EngineCoreOutput>,
pub scheduler_stats: Option<Box<SchedulerStats>>,
pub timestamp: f64,
pub finished_requests: Option<BTreeSet<String>>,
}
#[derive(Debug, Clone, PartialEq)]
pub struct UtilityCallOutput {
pub engine_index: u32,
pub timestamp: f64,
pub output: UtilityOutput,
}
/// Semantic classification of a raw `EngineCoreOutputs` message.
///
/// Python currently uses one product-shaped wire struct for several distinct
/// output families. This enum exposes those families more explicitly without
/// changing the wire format.
#[derive(Debug, Clone, PartialEq, EnumAsInner)]
pub enum ClassifiedEngineCoreOutputs {
RequestBatch(RequestBatchOutputs),
Utility(UtilityCallOutput),
DpControl {
engine_index: u32,
timestamp: f64,
control: DpControlMessage,
},
/// Fallback for wire-shape combinations that do not map cleanly onto the
/// current semantic families.
Other(EngineCoreOutputs),
}
impl EngineCoreOutputs {
/// Classify the raw wire message into a more semantic Rust enum.
pub fn classify(self) -> ClassifiedEngineCoreOutputs {
let has_request_payload = !self.outputs.is_empty()
|| self.scheduler_stats.is_some()
|| self.finished_requests.is_some();
match (
has_request_payload,
&self.utility_output,
&self.wave_complete,
&self.start_wave,
) {
(true, None, None, None) => {
ClassifiedEngineCoreOutputs::RequestBatch(RequestBatchOutputs {
engine_index: self.engine_index,
outputs: self.outputs,
scheduler_stats: self.scheduler_stats,
timestamp: self.timestamp,
finished_requests: self.finished_requests,
})
}
(false, Some(_), None, None) => {
ClassifiedEngineCoreOutputs::Utility(UtilityCallOutput {
engine_index: self.engine_index,
timestamp: self.timestamp,
output: self.utility_output.unwrap(),
})
}
(false, None, Some(_), None) => ClassifiedEngineCoreOutputs::DpControl {
engine_index: self.engine_index,
timestamp: self.timestamp,
control: DpControlMessage::WaveComplete(self.wave_complete.unwrap()),
},
(false, None, None, Some(_)) => ClassifiedEngineCoreOutputs::DpControl {
engine_index: self.engine_index,
timestamp: self.timestamp,
control: DpControlMessage::StartWave(self.start_wave.unwrap()),
},
_ => ClassifiedEngineCoreOutputs::Other(self),
}
}
}
#[cfg(test)]
mod tests {
use std::collections::BTreeSet;
use super::*;
use crate::protocol::EngineCoreOutput;
#[test]
fn engine_core_outputs_classify_request_batch() {
let outputs = EngineCoreOutputs {
outputs: vec![EngineCoreOutput {
request_id: "req-1".to_string(),
new_token_ids: vec![7],
..Default::default()
}],
finished_requests: Some(BTreeSet::from(["req-1".to_string()])),
..Default::default()
};
expect_test::expect![[r#"
RequestBatch(
RequestBatchOutputs {
engine_index: 0,
outputs: [
EngineCoreOutput {
request_id: "req-1",
new_token_ids: [
7,
],
new_logprobs: None,
new_prompt_logprobs_tensors: None,
pooling_output: None,
finish_reason: None,
stop_reason: None,
events: None,
kv_transfer_params: None,
trace_headers: None,
prefill_stats: None,
routed_experts: None,
num_nans_in_logits: 0,
},
],
scheduler_stats: None,
timestamp: 0.0,
finished_requests: Some(
{
"req-1",
},
),
},
)
"#]]
.assert_debug_eq(&outputs.classify());
}
#[test]
fn engine_core_outputs_classify_utility() {
let outputs = EngineCoreOutputs {
utility_output: Some(UtilityOutput {
call_id: 42,
failure_message: None,
result: None,
}),
..Default::default()
};
expect_test::expect![[r#"
Utility(
UtilityCallOutput {
engine_index: 0,
timestamp: 0.0,
output: UtilityOutput {
call_id: 42,
failure_message: None,
result: None,
},
},
)
"#]]
.assert_debug_eq(&outputs.classify());
}
#[test]
fn engine_core_outputs_classify_control() {
let outputs = EngineCoreOutputs {
start_wave: Some(3),
..Default::default()
};
expect_test::expect![[r#"
DpControl {
engine_index: 0,
timestamp: 0.0,
control: StartWave(
3,
),
}
"#]]
.assert_debug_eq(&outputs.classify());
}
#[test]
fn engine_core_outputs_classify_mixed_shape_as_raw() {
let outputs = EngineCoreOutputs {
outputs: vec![EngineCoreOutput {
request_id: "req-1".to_string(),
new_token_ids: vec![7],
..Default::default()
}],
utility_output: Some(UtilityOutput {
call_id: 1,
failure_message: None,
result: None,
}),
..Default::default()
};
expect_test::expect![[r#"
Other(
EngineCoreOutputs {
engine_index: 0,
outputs: [
EngineCoreOutput {
request_id: "req-1",
new_token_ids: [
7,
],
new_logprobs: None,
new_prompt_logprobs_tensors: None,
pooling_output: None,
finish_reason: None,
stop_reason: None,
events: None,
kv_transfer_params: None,
trace_headers: None,
prefill_stats: None,
routed_experts: None,
num_nans_in_logits: 0,
},
],
scheduler_stats: None,
timestamp: 0.0,
utility_output: Some(
UtilityOutput {
call_id: 1,
failure_message: None,
result: None,
},
),
finished_requests: None,
wave_complete: None,
start_wave: None,
},
)
"#]]
.assert_debug_eq(&outputs.classify());
}
}
@@ -0,0 +1,40 @@
use serde::{Deserialize, Serialize};
/// Effective model dtype reported by the engine after config resolution.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum ModelDtype {
#[serde(rename = "float16")]
Float16,
#[serde(rename = "bfloat16")]
BFloat16,
#[serde(rename = "float32")]
Float32,
}
impl ModelDtype {
pub fn as_str(self) -> &'static str {
match self {
Self::Float16 => "float16",
Self::BFloat16 => "bfloat16",
Self::Float32 => "float32",
}
}
}
#[cfg(test)]
mod tests {
use super::ModelDtype;
#[test]
fn serde_uses_protocol_dtype_strings() {
assert_eq!(
serde_json::to_value(ModelDtype::Float16).unwrap(),
serde_json::json!("float16")
);
assert_eq!(
serde_json::from_value::<ModelDtype>(serde_json::json!("bfloat16")).unwrap(),
ModelDtype::BFloat16
);
assert_eq!(ModelDtype::Float32.as_str(), "float32");
}
}
@@ -0,0 +1,90 @@
use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
use crate::protocol::{ModelDtype, OpaqueValue};
/// Decoded engine startup-handshake payload sent on the handshake socket.
///
/// Original Python payload construction:
/// <https://github.com/vllm-project/vllm/blob/c8d98f81f6/vllm/v1/engine/core.py#L1000-L1035>
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct ReadyMessage {
#[serde(default)]
pub status: Option<String>,
#[serde(default)]
pub local: Option<bool>,
#[serde(default)]
pub headless: Option<bool>,
#[serde(default)]
pub parallel_config_hash: Option<String>,
}
/// Post-initialization configuration sent from each engine on the input socket
/// registration message, after the handshake completes.
///
/// Contains values that may differ from the original config (e.g.
/// `max_model_len` after KV cache auto-fitting, `num_gpu_blocks` after
/// profiling).
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/c8d98f81f6/vllm/v1/engine/__init__.py#L67-L77>
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EngineCoreReadyResponse {
/// Engine-reported maximum model context length (auto-fitted after
/// KV cache profiling and may differ from the original config value).
pub max_model_len: u64,
/// Number of GPU blocks available for KV cache on this engine.
pub num_gpu_blocks: u64,
/// DP coordinator stats publish address, if applicable.
pub dp_stats_address: Option<String>,
/// Effective model dtype after Python vLLM resolves `--dtype`.
// TODO: This is currently not wired up on the engine side. After it's added, remove `Option`
// and `serde(default)`.
#[serde(default)]
pub dtype: Option<ModelDtype>,
}
/// Frontend-owned ZMQ addresses that are sent to the engine during startup
/// handshake initialization.
///
/// Original Python definition (`EngineZmqAddresses`):
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/utils.py#L53-L67>
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct HandshakeAddresses {
pub inputs: Vec<String>,
pub outputs: Vec<String>,
pub coordinator_input: Option<String>,
pub coordinator_output: Option<String>,
pub frontend_stats_publish_address: Option<String>,
}
/// Startup handshake payload sent from the frontend to initialize an engine
/// after receiving `HELLO`.
///
/// Original Python definition (`EngineHandshakeMetadata`):
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/utils.py#L69-L77>
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct HandshakeInitMessage {
pub addresses: HandshakeAddresses,
pub parallel_config: BTreeMap<String, OpaqueValue>,
}
#[cfg(test)]
mod tests {
use super::EngineCoreReadyResponse;
use crate::protocol::ModelDtype;
#[test]
fn ready_response_accepts_effective_dtype() {
let response: EngineCoreReadyResponse = serde_json::from_value(serde_json::json!({
"max_model_len": 4096,
"num_gpu_blocks": 2,
"dp_stats_address": null,
"dtype": "bfloat16"
}))
.unwrap();
assert_eq!(response.dtype, Some(ModelDtype::BFloat16));
}
}
@@ -0,0 +1,330 @@
mod array;
#[cfg(test)]
mod tests;
mod wire;
use std::ops::{Deref, DerefMut};
use enum_as_inner::EnumAsInner;
use serde::{Deserialize, Deserializer, Serialize};
use self::wire::*;
use super::{EngineCoreOutput, EngineCoreOutputs, decode_msgpack};
use crate::error::{Error, Result, bail_ext_value_decode, ext_value_decode};
use crate::protocol::tensor::{WireArrayData, WireNdArray};
/// One token candidate and its logprob metadata for a single sequence position.
///
/// The first entry in a [`PositionLogprobs`] is always the sampled/selected
/// token for that position. Any remaining entries follow the engine's returned
/// top-k candidate order.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct TokenLogprob {
pub token_id: u32,
pub logprob: f32,
/// The sampled/selected token uses its actual vocab rank. Remaining entries
/// use 1-based top-k ranks matching the engine's returned candidate
/// order.
pub rank: u32,
}
/// Logprob payload for one sequence position.
///
/// This is the semantic Rust representation used by the public client API after
/// the lower-level ndarray/tensor wire payload has been decoded.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct PositionLogprobs {
pub entries: Vec<TokenLogprob>,
}
impl PositionLogprobs {
/// Convert one decoded logprobs row into this per-position form by grouping
/// each token/logprob pair together with the sampled/selected token's
/// actual vocab rank.
fn from_decoded_row(token_ids: &[u32], logprobs: &[f32], sampled_rank: u32) -> Result<Self> {
if token_ids.len() != logprobs.len() {
bail_ext_value_decode!(
"logprobs row length mismatch: token_ids={}, logprobs={}",
token_ids.len(),
logprobs.len()
);
}
if sampled_rank == 0 {
bail_ext_value_decode!("token_ranks must be >= 1 for decoded engine-core logprobs");
}
let mut entries = Vec::with_capacity(token_ids.len());
for (index, (&token_id, &logprob)) in token_ids.iter().zip(logprobs.iter()).enumerate() {
let rank = if index == 0 {
sampled_rank
} else {
index as u32
};
entries.push(TokenLogprob {
token_id,
logprob,
rank,
});
}
Ok(Self { entries })
}
}
/// Decoded per-request logprobs payload for one engine-core output.
///
/// Unlike the Python wire payload, this public Rust type is already fully
/// semantic: one [`PositionLogprobs`] per scored position, each containing the
/// sampled/selected token plus any returned top-k alternatives for that same
/// position.
///
/// The Python engine still sends logprobs as ndarray/tensor-shaped wire tuples.
/// Rust resolves that lower-level representation during decode and exposes only
/// this per-position form to callers.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Logprobs {
/// One decoded logprobs record per scored position in this engine-core
/// output.
pub positions: Vec<PositionLogprobs>,
}
impl Logprobs {
/// Returns the number of scored positions in this payload.
pub fn len(&self) -> usize {
self.positions.len()
}
/// Returns whether the payload contains no scored positions.
pub fn is_empty(&self) -> bool {
self.positions.is_empty()
}
}
/// Output field wrapper that is initially deserialized from the Python wire
/// shape, then resolved into [`Logprobs`] before the decoded message is
/// returned to callers.
#[derive(Clone, PartialEq, Debug, EnumAsInner)]
pub enum MaybeWireLogprobs {
/// The logprobs are still in the wire format and need to be resolved by
/// looking up aux frames and decoding raw views. Should only be used
/// internally during deserialization.
Wire(Box<WireLogprobs>),
/// The actual decoded logprobs value,
Direct(Logprobs),
}
impl Deref for MaybeWireLogprobs {
type Target = Logprobs;
fn deref(&self) -> &Self::Target {
match self {
Self::Wire(_) => panic!("Logprobs is still in wire format"),
Self::Direct(value) => value,
}
}
}
impl DerefMut for MaybeWireLogprobs {
fn deref_mut(&mut self) -> &mut Self::Target {
match self {
Self::Wire(_) => panic!("Logprobs is still in wire format"),
Self::Direct(value) => value,
}
}
}
impl<'de> Deserialize<'de> for MaybeWireLogprobs {
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
where
D: Deserializer<'de>,
{
// When deserializing, it's always in the wire form.
WireLogprobs::deserialize(deserializer).map(|v| Self::Wire(Box::new(v)))
}
}
impl Serialize for MaybeWireLogprobs {
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
// For testing purposes only. We don't actually serialize it into aux frames.
match self {
Self::Wire(value) => value.serialize(serializer),
Self::Direct(value) => WireLogprobs::from_direct(value)
.map_err(serde::ser::Error::custom)?
.serialize(serializer),
}
}
}
impl MaybeWireLogprobs {
/// Resolve the wire representation into decoded logprobs by looking up aux
/// frames and decoding raw views as needed.
fn resolve<Frame>(self, frames: &[Frame], field_prefix: &str) -> Result<Self>
where
Frame: AsRef<[u8]>,
{
match self {
Self::Direct(value) => Ok(Self::Direct(value)),
Self::Wire(value) => value.resolve(frames, field_prefix).map(Self::Direct),
}
}
}
impl EngineCoreOutputs {
/// Resolve all wire-format fields in-place by looking up aux frames and
/// decoding raw-view payloads as needed.
fn resolve_in_place<Frame>(&mut self, frames: &[Frame]) -> Result<()>
where
Frame: AsRef<[u8]>,
{
for output in &mut self.outputs {
output.resolve_in_place(frames)?;
}
Ok(())
}
}
impl EngineCoreOutput {
/// Resolve all wire-format fields in-place by looking up aux frames and
/// decoding raw-view payloads as needed.
fn resolve_in_place<Frame>(&mut self, frames: &[Frame]) -> Result<()>
where
Frame: AsRef<[u8]>,
{
self.new_logprobs = (self.new_logprobs.take())
.map(|value| value.resolve(frames, "new_logprobs"))
.transpose()?;
self.new_prompt_logprobs_tensors = (self.new_prompt_logprobs_tensors.take())
.map(|value| value.resolve(frames, "new_prompt_logprobs_tensors"))
.transpose()?;
Ok(())
}
}
impl WireLogprobs {
/// Convert semantic per-position logprobs into the Python wire tuple shape.
///
/// This exists mainly so Rust-side tests can inject semantic logprobs into
/// mocked engine-core outputs without manually building ndarray
/// raw-view tuples.
fn from_direct(value: &Logprobs) -> std::result::Result<Self, String> {
let rows = value.positions.len();
let cols = value.positions.first().map(|position| position.entries.len()).unwrap_or(0);
let mut token_ids = Vec::with_capacity(rows.saturating_mul(cols).saturating_mul(8));
let mut logprobs = Vec::with_capacity(rows.saturating_mul(cols).saturating_mul(4));
let mut token_ranks = Vec::with_capacity(rows.saturating_mul(8));
for (row_index, position) in value.positions.iter().enumerate() {
if position.entries.len() != cols {
return Err(format!(
"logprobs row {row_index} length mismatch: expected {cols}, got {}",
position.entries.len()
));
}
let Some((sampled, _)) = position.entries.split_first() else {
return Err(format!("logprobs row {row_index} is empty"));
};
token_ranks.extend_from_slice(&(sampled.rank as i64).to_le_bytes());
for entry in &position.entries {
token_ids.extend_from_slice(&(entry.token_id as i64).to_le_bytes());
logprobs.extend_from_slice(&entry.logprob.to_le_bytes());
}
}
Ok(Self {
logprob_token_ids: WireNdArray {
dtype: "<i8".to_string(),
shape: vec![rows, cols],
data: WireArrayData::RawView(token_ids),
},
logprobs: WireNdArray {
dtype: "<f4".to_string(),
shape: vec![rows, cols],
data: WireArrayData::RawView(logprobs),
},
token_ranks: WireNdArray {
dtype: "<i8".to_string(),
shape: vec![rows],
data: WireArrayData::RawView(token_ranks),
},
cu_num_generated_tokens: None,
})
}
/// Resolve the wire-format logprobs into semantic [`Logprobs`] records by
/// looking up aux frames, decoding raw views, and grouping each row
/// into one [`PositionLogprobs`].
fn resolve<Frame>(self, frames: &[Frame], field_prefix: &str) -> Result<Logprobs>
where
Frame: AsRef<[u8]>,
{
if let Some(indices) = self.cu_num_generated_tokens {
bail_ext_value_decode!(
"{field_prefix}.cu_num_generated_tokens: \
expected None for per-request engine-core logprobs payload, got {indices:?}"
);
}
let token_ids = array::decode_array2_u32(
self.logprob_token_ids,
&format!("{field_prefix}.logprob_token_ids"),
frames,
)?;
let logprobs =
array::decode_array2_f32(self.logprobs, &format!("{field_prefix}.logprobs"), frames)?;
let token_ranks = array::decode_array1_u32(
self.token_ranks,
&format!("{field_prefix}.token_ranks"),
frames,
)?;
if token_ids.rows != logprobs.rows || token_ids.cols != logprobs.cols {
bail_ext_value_decode!(
"{field_prefix}: row shape mismatch between token ids ({}, {}) and logprobs ({}, {})",
token_ids.rows,
token_ids.cols,
logprobs.rows,
logprobs.cols
);
}
if token_ids.rows != token_ranks.len() {
bail_ext_value_decode!(
"{field_prefix}: token_ranks length {} does not match row count {}",
token_ranks.len(),
token_ids.rows
);
}
let mut positions = Vec::with_capacity(token_ids.rows);
for ((token_ids_row, logprobs_row), sampled_rank) in token_ids
.data
.chunks(token_ids.cols)
.zip(logprobs.data.chunks(logprobs.cols))
.zip(token_ranks)
{
positions.push(PositionLogprobs::from_decoded_row(
token_ids_row,
logprobs_row,
sampled_rank,
)?);
}
Ok(Logprobs { positions })
}
}
/// Decode one ordinary or multipart engine-core output message into the strong
/// typed public protocol shape.
pub fn decode_engine_core_outputs<Frame>(frames: &[Frame]) -> Result<EngineCoreOutputs>
where
Frame: AsRef<[u8]>,
{
let first_frame = frames.first().ok_or_else(|| ext_value_decode!("missing output frame"))?;
let mut outputs: EngineCoreOutputs = decode_msgpack(first_frame.as_ref())?;
outputs.resolve_in_place(frames)?;
Ok(outputs)
}
@@ -0,0 +1,306 @@
use std::io::Cursor;
use byteorder::{BigEndian, LittleEndian, NativeEndian, ReadBytesExt};
use itertools::Itertools as _;
use crate::error::{Error, Result, ext_value_decode};
use crate::protocol::tensor::{ShapeExt as _, WireArrayData, WireNdArray};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) enum ScalarType {
I32,
I64,
F32,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) enum Endianness {
Little,
Big,
Native,
}
#[derive(Debug, Clone, PartialEq)]
pub(super) struct DecodedArray2<T> {
pub rows: usize,
pub cols: usize,
pub data: Vec<T>,
}
pub(super) fn decode_array2_u32<Frame>(
value: WireNdArray,
field: &str,
frames: &[Frame],
) -> Result<DecodedArray2<u32>>
where
Frame: AsRef<[u8]>,
{
let (shape, bytes, scalar, endianness) =
decode_array_metadata(value, field, frames, &[ScalarType::I32, ScalarType::I64])?;
if shape.len() != 2 {
return Err(decode_error(
field,
&format!("expected rank-2 array, got rank {}", shape.len()),
));
}
let data = match scalar {
ScalarType::I32 => decode_i32_vec(&bytes, endianness, field)?
.into_iter()
.map(|value| convert_to_u32(value, field))
.try_collect()?,
ScalarType::I64 => decode_i64_vec(&bytes, endianness, field)?
.into_iter()
.map(|value| convert_to_u32(value, field))
.try_collect()?,
ScalarType::F32 => unreachable!("scalar validation should reject f32"),
};
Ok(DecodedArray2 {
rows: shape[0],
cols: shape[1],
data,
})
}
pub(super) fn decode_array1_u32<Frame>(
value: WireNdArray,
field: &str,
frames: &[Frame],
) -> Result<Vec<u32>>
where
Frame: AsRef<[u8]>,
{
let (shape, bytes, scalar, endianness) =
decode_array_metadata(value, field, frames, &[ScalarType::I32, ScalarType::I64])?;
if shape.len() != 1 {
return Err(decode_error(
field,
&format!("expected rank-1 array, got rank {}", shape.len()),
));
}
let data = match scalar {
ScalarType::I32 => decode_i32_vec(&bytes, endianness, field)?
.into_iter()
.map(|value| convert_to_u32(value, field))
.try_collect()?,
ScalarType::I64 => decode_i64_vec(&bytes, endianness, field)?
.into_iter()
.map(|value| convert_to_u32(value, field))
.try_collect()?,
ScalarType::F32 => unreachable!("scalar validation should reject f32"),
};
Ok(data)
}
pub(super) fn decode_array2_f32<Frame>(
value: WireNdArray,
field: &str,
frames: &[Frame],
) -> Result<DecodedArray2<f32>>
where
Frame: AsRef<[u8]>,
{
let (shape, bytes, _, endianness) =
decode_array_metadata(value, field, frames, &[ScalarType::F32])?;
if shape.len() != 2 {
return Err(decode_error(
field,
&format!("expected rank-2 array, got rank {}", shape.len()),
));
}
let data = decode_f32_vec(&bytes, endianness, field)?;
Ok(DecodedArray2 {
rows: shape[0],
cols: shape[1],
data,
})
}
pub(super) fn decode_array_metadata<Frame>(
value: WireNdArray,
field: &str,
frames: &[Frame],
expected_scalars: &[ScalarType],
) -> Result<(Vec<usize>, Vec<u8>, ScalarType, Endianness)>
where
Frame: AsRef<[u8]>,
{
let WireNdArray { dtype, shape, data } = value;
let (scalar, endianness) = parse_dtype(&dtype, field)?;
if !expected_scalars.contains(&scalar) {
return Err(decode_error(
field,
&format!("expected dtype in {:?}, got {}", expected_scalars, dtype),
));
}
let bytes = resolve_array_bytes(data, field, frames)?;
validate_byte_length(shape.as_slice(), bytes.len(), field, scalar)?;
Ok((shape, bytes, scalar, endianness))
}
pub(super) fn parse_dtype(dtype: &str, field: &str) -> Result<(ScalarType, Endianness)> {
let (endianness, body) = match dtype.as_bytes().first().copied() {
Some(b'<') => (Endianness::Little, &dtype[1..]),
Some(b'>') => (Endianness::Big, &dtype[1..]),
Some(b'=') => (Endianness::Native, &dtype[1..]),
Some(b'|') => (Endianness::Native, &dtype[1..]),
_ => (Endianness::Native, dtype),
};
let scalar = match body {
"i4" | "int32" => ScalarType::I32,
"i8" | "int64" => ScalarType::I64,
"f4" | "float32" => ScalarType::F32,
_ => {
return Err(decode_error(
field,
&format!("unsupported dtype string {dtype:?}"),
));
}
};
Ok((scalar, endianness))
}
pub(super) fn resolve_array_bytes<Frame>(
value: WireArrayData,
field: &str,
frames: &[Frame],
) -> Result<Vec<u8>>
where
Frame: AsRef<[u8]>,
{
match value {
WireArrayData::RawView(bytes) => Ok(bytes),
WireArrayData::AuxIndex(index) => {
let frame = frames.get(index).ok_or_else(|| {
decode_error(
field,
&format!(
"aux frame index {index} out of range for {} frames",
frames.len()
),
)
})?;
Ok(frame.as_ref().to_vec())
}
}
}
pub(super) fn validate_byte_length(
shape: &[usize],
byte_len: usize,
field: &str,
scalar: ScalarType,
) -> Result<()> {
let element_count = shape
.checked_numel()
.ok_or_else(|| decode_error(field, "shape element count overflowed usize"))?;
let element_size = match scalar {
ScalarType::I32 | ScalarType::F32 => 4,
ScalarType::I64 => 8,
};
let expected = element_count
.checked_mul(element_size)
.ok_or_else(|| decode_error(field, "byte length overflowed usize"))?;
if expected != byte_len {
return Err(decode_error(
field,
&format!("byte length mismatch: expected {expected}, got {byte_len}"),
));
}
Ok(())
}
pub(super) fn decode_i32_vec(
bytes: &[u8],
endianness: Endianness,
field: &str,
) -> Result<Vec<i32>> {
if !bytes.len().is_multiple_of(4) {
return Err(decode_error(
field,
&format!("byte length {} is not divisible by 4", bytes.len()),
));
}
let mut cursor = Cursor::new(bytes);
let mut values = Vec::with_capacity(bytes.len() / 4);
while (cursor.position() as usize) < bytes.len() {
let value = match endianness {
Endianness::Little => cursor.read_i32::<LittleEndian>(),
Endianness::Big => cursor.read_i32::<BigEndian>(),
Endianness::Native => cursor.read_i32::<NativeEndian>(),
}
.map_err(|error| decode_error(field, &format!("failed to read i32 payload: {error}")))?;
values.push(value);
}
Ok(values)
}
pub(super) fn decode_f32_vec(
bytes: &[u8],
endianness: Endianness,
field: &str,
) -> Result<Vec<f32>> {
if !bytes.len().is_multiple_of(4) {
return Err(decode_error(
field,
&format!("byte length {} is not divisible by 4", bytes.len()),
));
}
let mut cursor = Cursor::new(bytes);
let mut values = Vec::with_capacity(bytes.len() / 4);
while (cursor.position() as usize) < bytes.len() {
let value = match endianness {
Endianness::Little => cursor.read_f32::<LittleEndian>(),
Endianness::Big => cursor.read_f32::<BigEndian>(),
Endianness::Native => cursor.read_f32::<NativeEndian>(),
}
.map_err(|error| decode_error(field, &format!("failed to read f32 payload: {error}")))?;
values.push(value);
}
Ok(values)
}
pub(super) fn decode_i64_vec(
bytes: &[u8],
endianness: Endianness,
field: &str,
) -> Result<Vec<i64>> {
if !bytes.len().is_multiple_of(8) {
return Err(decode_error(
field,
&format!("byte length {} is not divisible by 8", bytes.len()),
));
}
let mut cursor = Cursor::new(bytes);
let mut values = Vec::with_capacity(bytes.len() / 8);
while (cursor.position() as usize) < bytes.len() {
let value = match endianness {
Endianness::Little => cursor.read_i64::<LittleEndian>(),
Endianness::Big => cursor.read_i64::<BigEndian>(),
Endianness::Native => cursor.read_i64::<NativeEndian>(),
}
.map_err(|error| decode_error(field, &format!("failed to read i64 payload: {error}")))?;
values.push(value);
}
Ok(values)
}
fn convert_to_u32<I>(value: I, field: &str) -> Result<u32>
where
I: TryInto<u32> + std::fmt::Display + Copy,
{
value.try_into().map_err(|_| {
decode_error(
field,
&format!("expected non-negative token id/rank that fits in u32, got {value}"),
)
})
}
pub(super) fn decode_error(field: &str, reason: &str) -> Error {
ext_value_decode!("{field}: {reason}")
}
@@ -0,0 +1,302 @@
use std::collections::BTreeSet;
use bytes::Bytes;
use rmpv::Value;
use super::{Logprobs, PositionLogprobs, TokenLogprob, decode_engine_core_outputs};
use crate::protocol::EngineCoreFinishReason;
fn encode_value(value: &Value) -> Vec<u8> {
let mut out = Vec::new();
rmpv::encode::write_value(&mut out, value).unwrap();
out
}
fn output_wire_with_custom_fields(
new_logprobs: Option<Value>,
prompt_logprobs: Option<Value>,
) -> Value {
Value::Array(vec![
Value::from(0),
Value::Array(vec![Value::Array(vec![
Value::from("req-1"),
Value::Array(vec![Value::from(7), Value::from(8)]),
new_logprobs.unwrap_or(Value::Nil),
prompt_logprobs.unwrap_or(Value::Nil),
Value::Nil,
Value::from(EngineCoreFinishReason::Length as u8),
])]),
Value::Nil,
Value::from(0.0),
Value::Nil,
Value::Array(vec![Value::from("req-1")]),
])
}
fn ndarray_value(dtype: &str, shape: &[usize], data: Value) -> Value {
Value::Array(vec![
Value::from(dtype),
Value::Array(shape.iter().copied().map(Value::from).collect()),
data,
])
}
fn inline_logprobs_value() -> Value {
let ids = Value::Ext(
3,
vec![
1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0,
0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0,
],
);
let probs = Value::Ext(
3,
vec![
0, 0, 128, 63, 0, 0, 0, 64, 0, 0, 64, 64, 0, 0, 128, 64, 0, 0, 160, 64, 0, 0, 192, 64,
],
);
let ranks = Value::Ext(3, vec![1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0]);
Value::Array(vec![
ndarray_value("<i8", &[2, 3], ids),
ndarray_value("<f4", &[2, 3], probs),
ndarray_value("<i8", &[2], ranks),
Value::Nil,
])
}
fn inline_prompt_logprobs_value() -> Value {
let ids = Value::Ext(
3,
vec![
10, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0,
0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0,
],
);
let probs = Value::Ext(
3,
vec![
0, 0, 32, 65, 0, 0, 48, 65, 0, 0, 64, 65, 0, 0, 80, 65, 0, 0, 96, 65, 0, 0, 112, 65,
],
);
let ranks = Value::Ext(3, vec![3, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0]);
Value::Array(vec![
ndarray_value("int64", &[2, 3], ids),
ndarray_value("float32", &[2, 3], probs),
ndarray_value("int64", &[2], ranks),
Value::Nil,
])
}
fn expected_sample_logprobs() -> Logprobs {
Logprobs {
positions: vec![
PositionLogprobs {
entries: vec![
TokenLogprob {
token_id: 1,
logprob: 1.0,
rank: 1,
},
TokenLogprob {
token_id: 2,
logprob: 2.0,
rank: 1,
},
TokenLogprob {
token_id: 3,
logprob: 3.0,
rank: 2,
},
],
},
PositionLogprobs {
entries: vec![
TokenLogprob {
token_id: 4,
logprob: 4.0,
rank: 2,
},
TokenLogprob {
token_id: 5,
logprob: 5.0,
rank: 1,
},
TokenLogprob {
token_id: 6,
logprob: 6.0,
rank: 2,
},
],
},
],
}
}
fn expected_prompt_logprobs() -> Logprobs {
Logprobs {
positions: vec![
PositionLogprobs {
entries: vec![
TokenLogprob {
token_id: 10,
logprob: 10.0,
rank: 3,
},
TokenLogprob {
token_id: 11,
logprob: 11.0,
rank: 1,
},
TokenLogprob {
token_id: 12,
logprob: 12.0,
rank: 2,
},
],
},
PositionLogprobs {
entries: vec![
TokenLogprob {
token_id: 13,
logprob: 13.0,
rank: 4,
},
TokenLogprob {
token_id: 14,
logprob: 14.0,
rank: 1,
},
TokenLogprob {
token_id: 15,
logprob: 15.0,
rank: 2,
},
],
},
],
}
}
#[test]
fn decodes_inline_new_logprobs() {
let frames = vec![Bytes::from(encode_value(&output_wire_with_custom_fields(
Some(inline_logprobs_value()),
None,
)))];
let decoded = decode_engine_core_outputs(&frames).unwrap();
let logprobs = decoded.outputs[0].new_logprobs.clone().unwrap().into_direct().unwrap();
assert_eq!(logprobs, expected_sample_logprobs());
assert_eq!(
decoded.finished_requests,
Some(BTreeSet::from(["req-1".to_string()]))
);
}
#[test]
fn decodes_multipart_new_logprobs() {
let frames = vec![
Bytes::from(encode_value(&output_wire_with_custom_fields(
Some(Value::Array(vec![
ndarray_value("<i8", &[2, 3], Value::from(1)),
ndarray_value("<f4", &[2, 3], Value::from(2)),
ndarray_value("<i8", &[2], Value::from(3)),
Value::Nil,
])),
None,
))),
Bytes::from_static(&[
1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0,
0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0,
]),
Bytes::from_static(&[
0, 0, 128, 63, 0, 0, 0, 64, 0, 0, 64, 64, 0, 0, 128, 64, 0, 0, 160, 64, 0, 0, 192, 64,
]),
Bytes::from_static(&[1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0]),
];
let decoded = decode_engine_core_outputs(&frames).unwrap();
let logprobs = decoded.outputs[0].new_logprobs.clone().unwrap().into_direct().unwrap();
assert_eq!(logprobs, expected_sample_logprobs());
}
#[test]
fn decodes_inline_prompt_logprobs() {
let frames = vec![Bytes::from(encode_value(&output_wire_with_custom_fields(
None,
Some(inline_prompt_logprobs_value()),
)))];
let decoded = decode_engine_core_outputs(&frames).unwrap();
let logprobs = decoded.outputs[0]
.new_prompt_logprobs_tensors
.clone()
.unwrap()
.into_direct()
.unwrap();
assert_eq!(logprobs, expected_prompt_logprobs());
}
#[test]
fn decodes_big_endian_payloads() {
let frames = vec![Bytes::from(encode_value(&output_wire_with_custom_fields(
Some(Value::Array(vec![
ndarray_value(">i4", &[1, 2], Value::Ext(3, vec![0, 0, 0, 1, 0, 0, 0, 2])),
ndarray_value(
">f4",
&[1, 2],
Value::Ext(3, vec![63, 128, 0, 0, 64, 0, 0, 0]),
),
ndarray_value(">i4", &[1], Value::Ext(3, vec![0, 0, 0, 3])),
Value::Nil,
])),
None,
)))];
let decoded = decode_engine_core_outputs(&frames).unwrap();
let logprobs = decoded.outputs[0].new_logprobs.clone().unwrap().into_direct().unwrap();
assert_eq!(
logprobs,
Logprobs {
positions: vec![PositionLogprobs {
entries: vec![
TokenLogprob {
token_id: 1,
logprob: 1.0,
rank: 3,
},
TokenLogprob {
token_id: 2,
logprob: 2.0,
rank: 1,
},
],
}],
}
);
}
#[test]
fn rejects_non_none_cu_num_generated_tokens() {
let frames = vec![Bytes::from(encode_value(&output_wire_with_custom_fields(
Some(Value::Array(vec![
ndarray_value("<i8", &[1, 1], Value::Ext(3, vec![1, 0, 0, 0, 0, 0, 0, 0])),
ndarray_value("<f4", &[1, 1], Value::Ext(3, vec![0, 0, 128, 63])),
ndarray_value("<i8", &[1], Value::Ext(3, vec![1, 0, 0, 0, 0, 0, 0, 0])),
Value::Array(vec![Value::from(0usize), Value::from(1usize)]),
])),
None,
)))];
let error = decode_engine_core_outputs(&frames).unwrap_err();
let crate::error::Error::ExtValueDecode { message } = &error else {
panic!("expected ValueDecodeExt");
};
assert_eq!(
message,
"new_logprobs.cu_num_generated_tokens: expected None for per-request engine-core logprobs payload, got [0, 1]"
);
assert_eq!(
error.to_string(),
"messagepack ext value decode failed: new_logprobs.cu_num_generated_tokens: expected None for per-request engine-core logprobs payload, got [0, 1]"
);
}
@@ -0,0 +1,31 @@
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
use crate::protocol::tensor::WireNdArray;
/// Python wire representation of `LogprobsLists` / `LogprobsTensors` before
/// aux-frame references and raw-view payloads are resolved.
///
/// This mirrors the tuple shape emitted by Python engine-core so serde can
/// first deserialize the raw wire payload before the Rust client converts it
/// into semantic per-position logprobs records.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/outputs.py#L23-L56>
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple)]
pub struct WireLogprobs {
/// Wire array with shape `[num_positions, max_num_logprobs + 1]`.
pub logprob_token_ids: WireNdArray,
/// Wire array with shape `[num_positions, max_num_logprobs + 1]`.
pub logprobs: WireNdArray,
/// Wire array with shape `[num_positions]`.
///
/// Python uses the field name `sampled_token_ranks` for sample logprobs and
/// `selected_token_ranks` for prompt logprobs. Rust keeps one neutral field
/// because both payloads share the same wire representation.
pub token_ranks: WireNdArray,
/// Preserved only for wire compatibility with batch-level Python tensors.
/// Scheduler-sliced per-request outputs should emit `None` here, and
/// the semantic Rust decoder rejects any other value.
#[serde(default)]
pub cu_num_generated_tokens: Option<Vec<usize>>,
}
@@ -0,0 +1,756 @@
use std::any::type_name;
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::io::Cursor;
use bytes::Bytes;
use rmpv::Value;
use serde::{Deserialize, Serialize};
use serde_default::DefaultFromSerde;
use serde_repr::{Deserialize_repr, Serialize_repr};
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
use thiserror_ext::AsReport;
use crate::error::{Error, Result};
use crate::protocol::logprobs::MaybeWireLogprobs;
use crate::protocol::multimodal::MmFeatures;
use crate::protocol::stats::{PrefillStats, SchedulerStats};
// TODO: This module currently mixes reusable frontend-facing semantic types
// (for example `FinishReason`, `StopReason`, `RequestOutputKind`, and future
// cleaned-up frontend sampling types) with engine-core-specific wire DTOs and
// handshake/control messages. While the Rust frontend is still evolving
// quickly, keep them co-located here for iteration speed. Once the higher-level
// API boundary stabilizes, move the truly reusable semantic types into a
// lower-level common crate and keep the engine transport/wire messages here.
/// Dynamic msgpack value used for schema positions that are preserved but not
/// yet strongly typed in the early-stage Rust client.
pub type OpaqueValue = Value;
fn default_opaque_value_nil() -> OpaqueValue {
Value::Nil
}
fn is_false(v: &bool) -> bool {
!v
}
mod classified_outputs;
pub mod dtype;
pub mod handshake;
pub mod logprobs;
pub mod multimodal;
pub mod stats;
pub mod tensor;
pub use classified_outputs::{
ClassifiedEngineCoreOutputs, DpControlMessage, RequestBatchOutputs, UtilityCallOutput,
};
pub use dtype::ModelDtype;
pub use logprobs::decode_engine_core_outputs;
/// Request types are encoded as single-byte protocol constants so they can be
/// sent over the ZMQ socket without an extra encoding step.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L217-L228>
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[repr(u8)]
pub enum EngineCoreRequestType {
Add = 0,
Abort = 1,
StartDpWave = 2,
Utility = 3,
}
impl EngineCoreRequestType {
pub fn to_frame(self) -> Bytes {
Bytes::from_static(match self {
Self::Add => b"\x00",
Self::Abort => b"\x01",
Self::StartDpWave => b"\x02",
Self::Utility => b"\x03",
})
}
}
/// Reason a request finished: stop, length, abort, error, or repetition.
///
/// This mirrors the Python enum and uses integer encoding for compact wire
/// representation.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L41-L63>
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize_repr, Deserialize_repr)]
#[repr(u8)]
pub enum EngineCoreFinishReason {
/// A stop string was emitted.
Stop = 0,
/// `max_tokens` or `max_model_len` was reached.
Length = 1,
/// The request was aborted by the client.
Abort = 2,
/// A retryable request-level internal error occurred.
Error = 3,
/// A repetitive token pattern was detected.
Repetition = 4,
}
/// Event types emitted by engine-core for one request.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L113-L118>
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize_repr, Deserialize_repr)]
#[repr(u8)]
pub enum EngineCoreEventType {
Queued = 1,
Scheduled = 2,
Preempted = 3,
}
/// A timestamped engine-core event associated with one request.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L121-L130>
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct EngineCoreEvent {
pub r#type: EngineCoreEventType,
pub timestamp: f64,
}
/// Controls how intermediate outputs are returned to the frontend.
///
/// `Cumulative = 0` is intentionally not supported in Rust frontend.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/sampling_params.py#L146-L152>
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize_repr, Deserialize_repr)]
#[repr(u8)]
pub enum RequestOutputKind {
/// Return only token deltas in each update.
#[default]
Delta = 1,
/// Suppress intermediate updates and return only the final output.
FinalOnly = 2,
}
/// The stop reason associated with a finished output.
///
/// Python models this as the union-typed `stop_reason: int | str | None`
/// field on `EngineCoreOutput`; the Rust client narrows it into a tagged enum.
///
/// Original Python field:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L155>
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(untagged)]
pub enum StopReason {
TokenId(u32),
Text(String),
}
/// Parameters for configuring structured outputs (guided decoding).
///
/// Exactly one constraint field (`json`, `regex`, `choice`, `grammar`,
/// `json_object`, or `structural_tag`) should be set. The engine-core
/// backend selects the appropriate grammar compiler based on which field
/// is present.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/sampling_params.py#L36-L107>
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
#[serde(default)]
pub struct StructuredOutputsParams {
/// JSON schema (as a dict/object or JSON string) constraining the output.
pub json: Option<serde_json::Value>,
/// Regular expression the output must match.
pub regex: Option<String>,
/// List of allowed output strings (the model must produce one of these).
pub choice: Option<Vec<String>>,
/// Context-free grammar (in EBNF-like notation) the output must conform to.
pub grammar: Option<String>,
/// When `true`, output must be valid JSON (free-form, no schema).
pub json_object: Option<bool>,
/// Disable any additional whitespace in guided JSON output.
#[serde(skip_serializing_if = "crate::protocol::is_false")]
pub disable_any_whitespace: bool,
/// Disable `additionalProperties` in JSON schema output.
#[serde(skip_serializing_if = "crate::protocol::is_false")]
pub disable_additional_properties: bool,
/// Custom whitespace pattern for guided JSON output.
pub whitespace_pattern: Option<String>,
/// Structural tag configuration (JSON-encoded string).
pub structural_tag: Option<String>,
}
/// Engine-core-facing sampling parameters for text generation.
///
/// This is the normalized southbound subset used by the Rust frontend when it
/// talks to Python engine-core over the wire. User-facing request semantics
/// such as `stop` strings, `n`, `ignore_eos`, and output aggregation mode are
/// intentionally handled by higher layers before values reach this DTO.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/sampling_params.py#L155-L291>
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct EngineCoreSamplingParams {
/// Controls randomness. Lower values are more deterministic; zero means
/// greedy sampling.
pub temperature: f32,
/// Cumulative probability threshold for nucleus sampling.
pub top_p: f32,
/// Maximum number of top tokens to consider. `0` means all tokens.
pub top_k: u32,
/// Random seed used by the sampler when present.
pub seed: Option<i64>,
/// Maximum number of tokens to generate per output sequence.
pub max_tokens: u32,
/// Minimum number of tokens to generate before EOS or stop-token handling.
pub min_tokens: u32,
/// Number of log probabilities to return per generated token.
///
/// `None` disables sample logprobs. `-1` requests the full vocabulary.
pub logprobs: Option<i32>,
/// Number of log probabilities to return per prompt token.
///
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
pub prompt_logprobs: Option<i32>,
/// Minimum probability threshold for token sampling.
pub min_p: f32,
/// Frequency penalty applied by the sampler.
pub frequency_penalty: f32,
/// Presence penalty applied by the sampler.
pub presence_penalty: f32,
/// Repetition penalty applied by the sampler.
pub repetition_penalty: f32,
/// Token IDs that stop generation.
pub stop_token_ids: Vec<u32>,
/// Primary EOS token ID used by engine-core's dedicated EOS stop path.
///
/// This mirrors Python's internal `_eos_token_id` field and is derived by
/// the frontend from tokenizer/model metadata rather than supplied directly
/// by end users.
#[serde(rename = "_eos_token_id")]
pub eos_token_id: Option<u32>,
/// Complete stop-token set used by engine-core for `min_tokens` masking.
///
/// This mirrors Python's internal `_all_stop_token_ids` field and should
/// contain explicit `stop_token_ids` plus any frontend-derived EOS token
/// IDs.
#[serde(rename = "_all_stop_token_ids")]
pub all_stop_token_ids: BTreeSet<u32>,
/// Logit biases to apply during sampling.
/// Keys are token IDs
#[serde(default)]
pub logit_bias: Option<HashMap<u32, f32>>,
/// Restrict output to these token IDs only.
#[serde(default)]
pub allowed_token_ids: Option<Vec<u32>>,
/// Tokenized bad words to avoid during generation.
#[serde(default, rename = "_bad_words_token_ids")]
pub bad_words_token_ids: Option<Vec<Vec<u32>>>,
/// Parameters for configuring structured outputs (guided decoding).
#[serde(default)]
pub structured_outputs: Option<StructuredOutputsParams>,
/// Specific token IDs for which log probabilities should be returned at
/// each position.
///
/// When set, the engine returns logprobs for exactly these tokens in
/// addition to the sampled/scored token. Mutually exclusive with the
/// `logprobs` count field in practice.
#[serde(default)]
pub logprob_token_ids: Option<Vec<u32>>,
/// If `Some(true)`, the request will not attempt to read from the prefix
/// cache; newly computed blocks may still populate the cache. `None`
/// defers to engine-core defaults.
#[serde(default)]
pub skip_reading_prefix_cache: Option<bool>,
/// Additional request parameters for custom extensions (from `vllm_xargs`).
#[serde(default)]
pub extra_args: Option<HashMap<String, serde_json::Value>>,
}
impl EngineCoreSamplingParams {
/// Constructs a default sampling params for testing purposes only.
pub fn for_test() -> Self {
Self {
temperature: 1.0,
top_p: 1.0,
top_k: 0,
seed: None,
max_tokens: 65536,
min_tokens: 0,
logprobs: None,
prompt_logprobs: None,
min_p: 0.0,
frequency_penalty: 0.0,
presence_penalty: 0.0,
repetition_penalty: 1.0,
stop_token_ids: Vec::new(),
eos_token_id: None,
all_stop_token_ids: BTreeSet::new(),
logit_bias: None,
allowed_token_ids: None,
bad_words_token_ids: None,
structured_outputs: None,
logprob_token_ids: None,
skip_reading_prefix_cache: None,
extra_args: None,
}
}
}
/// Engine-core add-request payload sent from frontend to engine.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/3f5bd482f5c1a5dbdffbbf68d624e20bb7032013/vllm/v1/engine/__init__.py#L80-L129>
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple, DefaultFromSerde)]
pub struct EngineCoreRequest {
pub request_id: String,
pub prompt_token_ids: Option<Vec<u32>>,
/// Multimodal features attached to the request.
pub mm_features: Option<MmFeatures>,
pub sampling_params: Option<EngineCoreSamplingParams>,
/// Pooling parameters are preserved in the schema but not yet strongly
/// typed.
pub pooling_params: Option<OpaqueValue>,
pub arrival_time: f64,
#[serde(default)]
pub lora_request: Option<OpaqueValue>,
#[serde(default)]
pub cache_salt: Option<String>,
#[serde(default)]
pub data_parallel_rank: Option<u32>,
/// Unsupported in the first-stage Rust client because Python uses a custom
/// tensor/aux-frame encoding path for this field.
#[serde(default)]
pub prompt_embeds: Option<OpaqueValue>,
/// Per-position mask for mixed-mode inputs (e.g. chat completion with
/// `prompt_embeds` content parts). `Some(true)` means real token id;
/// `Some(false)` means the position uses a pre-computed entry from
/// `prompt_embeds`. `None` for pure-tokens and pure-embeds requests.
#[serde(default)]
pub prompt_is_token_ids: Option<Vec<bool>>,
/// Index of the client, used to ensure outputs are sent back to the same
/// client when scaling out the frontend.
#[serde(default)]
pub client_index: u32,
/// In DP mode, indicates which wave this request is expected to belong to.
#[serde(default)]
pub current_wave: u32,
#[serde(default)]
pub priority: i32,
#[serde(default)]
pub trace_headers: Option<BTreeMap<String, String>>,
#[serde(default)]
pub resumable: bool,
/// Original user-provided request ID, used for output reporting and aborts.
#[serde(default)]
pub external_req_id: Option<String>,
#[serde(default)]
pub reasoning_ended: Option<bool>,
/// Opaque reasoning-parser kwargs forwarded from the frontend to the
/// structured-output backend.
#[serde(default)]
pub reasoning_parser_kwargs: Option<OpaqueValue>,
/// If `true`, the request should be added to the scheduler's waiting queue
/// and immediately aborted, so connector-side cleanup runs via the
/// standard `request_finished` hook.
#[serde(default)]
pub abort_immediately: bool,
}
impl EngineCoreRequest {
/// Validate fields intentionally not supported in the first-stage client.
pub fn validate(&self) -> Result<()> {
if self.prompt_embeds.is_some() {
return Err(Error::UnsupportedField {
context: "EngineCoreRequest",
field: "prompt_embeds",
});
}
Ok(())
}
}
/// Engine-core utility call payload sent from frontend to engine.
///
/// Original Python payload shape:
/// `(client_index, call_id, method_name, args)`
#[derive(Debug, Clone, PartialEq, Serialize_tuple)]
pub struct EngineCoreUtilityRequest {
pub client_index: u32,
pub call_id: i64,
pub method_name: String,
pub args: OpaqueValue,
}
impl EngineCoreUtilityRequest {
/// Create a new utility request with the given strongly typed arguments,
/// encoding them into the expected msgpack value format.
pub fn new<T>(
client_index: u32,
call_id: i64,
method_name: impl Into<String>,
args: T,
) -> Result<Self>
where
T: Serialize + std::fmt::Debug,
{
let args = rmpv::ext::to_value(&args).map_err(|error| Error::Encode {
target_type: type_name::<T>(),
message: format!(
"failed to encode utility args `{args:?}`: {}",
error.to_report_string()
),
})?;
let args = match args {
Value::Nil => Value::Array(Vec::new()),
other => other,
};
Ok(Self {
client_index,
call_id,
method_name: method_name.into(),
args,
})
}
}
/// Engine-core output for a single request.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/d3af8c18317c0dc008d42e4367fbb9045cfb7bf6/vllm/v1/engine/__init__.py#L154-L184>
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple, DefaultFromSerde)]
pub struct EngineCoreOutput {
pub request_id: String,
pub new_token_ids: Vec<u32>,
/// Decoded sample logprobs for the newly generated positions in this
/// output.
#[serde(default)]
pub new_logprobs: Option<MaybeWireLogprobs>,
/// Decoded prompt logprobs for the scored prompt positions emitted in this
/// output.
#[serde(default)]
pub new_prompt_logprobs_tensors: Option<MaybeWireLogprobs>,
#[serde(default)]
pub pooling_output: Option<OpaqueValue>,
#[serde(default)]
pub finish_reason: Option<EngineCoreFinishReason>,
#[serde(default)]
pub stop_reason: Option<StopReason>,
#[serde(default)]
pub events: Option<Vec<EngineCoreEvent>>,
#[serde(default)]
pub kv_transfer_params: Option<serde_json::Value>,
#[serde(default)]
pub trace_headers: Option<OpaqueValue>,
/// Breakdown of the scheduled prefill computation, set on the first output
/// of a newly scheduled prefill and elided for subsequent decode outputs.
#[serde(default)]
pub prefill_stats: Option<PrefillStats>,
#[serde(default)]
pub routed_experts: Option<OpaqueValue>,
/// Number of NaNs seen in logits. Values above zero indicate corruption.
#[serde(default)]
pub num_nans_in_logits: u32,
}
impl EngineCoreOutput {
/// Returns whether this output is terminal for the request.
pub fn finished(&self) -> bool {
self.finish_reason.is_some()
}
}
/// Result of a utility call.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L174-L183>
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple, DefaultFromSerde)]
pub struct UtilityOutput {
pub call_id: i64,
/// Non-`None` implies the call failed and `result` should be ignored.
#[serde(default)]
pub failure_message: Option<String>,
#[serde(default)]
pub result: Option<UtilityResultEnvelope>,
}
/// Python `UtilityResult` wrapper carried inside `UtilityOutput.result`.
///
/// Upstream reference:
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/serial_utils.py#L178-L185>
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple)]
pub struct UtilityResultEnvelope {
/// Recursive type information encoded on Python side, serving as the hint
/// for deserialization. We don't care it here as in Rust frontend all
/// utility calls are strongly-typed.
#[serde(default)]
type_info: Option<OpaqueValue>,
/// The actual utility result.
#[serde(default = "default_opaque_value_nil")]
result: OpaqueValue,
}
impl UtilityResultEnvelope {
/// Create a utility result envelope without type information.
pub fn without_type_info(result: OpaqueValue) -> Self {
Self {
type_info: None,
result,
}
}
}
impl UtilityOutput {
/// Decode the typed result of a utility call.
pub fn into_typed_result<T>(self, method: &str) -> Result<T>
where
T: serde::de::DeserializeOwned,
{
if let Some(message) = self.failure_message {
return Err(Error::UtilityCallFailed {
method: method.to_string(),
call_id: self.call_id,
message,
});
}
let result = self.result.map(|e| e.result).unwrap_or(Value::Nil);
rmpv::ext::from_value(result).map_err(|error| Error::UtilityResultDecode {
method: method.to_string(),
call_id: self.call_id,
message: error.to_report_string(),
})
}
}
/// Batch of engine-core outputs returned to a frontend client.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L186-L214>
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple, DefaultFromSerde)]
pub struct EngineCoreOutputs {
#[serde(default)]
pub engine_index: u32,
/// Outputs grouped for this client in the current engine tick.
#[serde(default)]
pub outputs: Vec<EngineCoreOutput>,
#[serde(default)]
pub scheduler_stats: Option<Box<SchedulerStats>>,
#[serde(default)]
pub timestamp: f64,
#[serde(default)]
pub utility_output: Option<UtilityOutput>,
#[serde(default)]
pub finished_requests: Option<BTreeSet<String>>,
/// In DP mode, signals that the current wave finished and engines are
/// paused.
#[serde(default)]
pub wave_complete: Option<u32>,
/// In DP mode, signals that a request arrived for an old wave and the next
/// wave needs to start in other engines.
#[serde(default)]
pub start_wave: Option<u32>,
}
/// Encode a Rust value into msgpack using the protocol crate's serde model.
pub fn encode_msgpack<T>(value: &T) -> Result<Vec<u8>>
where
T: Serialize + std::fmt::Debug,
{
rmp_serde::to_vec_named(value).map_err(|error| Error::Encode {
target_type: type_name::<T>(),
message: format!(
"failed to encode value `{:?}`: {}",
value,
error.to_report_string()
),
})
}
/// Decode a msgpack payload into a strongly typed protocol value, with enhanced
/// error reporting.
pub fn decode_msgpack<T>(bytes: &[u8]) -> Result<T>
where
T: for<'de> Deserialize<'de>,
{
fn decode_value_preview(bytes: &[u8]) -> String {
match decode_value(bytes) {
Ok(value) => format!("{value}"),
Err(error) => format!("<value decode failed: {error}>"),
}
}
rmp_serde::from_slice(bytes).map_err(|error| Error::Decode {
target_type: type_name::<T>(),
message: format!("{error}; value fallback: {}", decode_value_preview(bytes)),
})
}
pub fn decode_value(bytes: &[u8]) -> Result<Value> {
Ok(rmpv::decode::read_value(&mut Cursor::new(bytes))?)
}
#[cfg(test)]
mod tests {
use std::collections::BTreeSet;
use super::*;
fn utility_result_value<T>(value: T) -> UtilityResultEnvelope
where
T: Serialize,
{
UtilityResultEnvelope::without_type_info(rmpv::ext::to_value(value).unwrap())
}
#[test]
fn engine_core_request_serializes_as_full_array() {
let request = EngineCoreRequest {
request_id: "req-1".to_string(),
prompt_token_ids: Some(vec![1, 2, 3]),
sampling_params: Some(EngineCoreSamplingParams {
max_tokens: 8,
..EngineCoreSamplingParams::for_test()
}),
arrival_time: 1234.5,
client_index: 7,
..EngineCoreRequest::default()
};
let encoded = encode_msgpack(&request).unwrap();
let value = decode_value(&encoded).unwrap();
let array = match value {
Value::Array(array) => array,
other => panic!("expected array, got {other:?}"),
};
assert_eq!(array.len(), 20);
assert_eq!(array[0], Value::from("req-1"));
assert_eq!(array[2], Value::Nil);
assert_eq!(array[4], Value::Nil);
assert_eq!(array[10], Value::Nil);
assert_eq!(array[11], Value::from(7));
}
#[test]
fn engine_core_outputs_roundtrip_finished_fields() {
let outputs = EngineCoreOutputs {
outputs: vec![EngineCoreOutput {
request_id: "req-1".to_string(),
new_token_ids: vec![42],
new_logprobs: None,
new_prompt_logprobs_tensors: None,
pooling_output: None,
finish_reason: Some(EngineCoreFinishReason::Length),
stop_reason: Some(StopReason::Text("stop".to_string())),
events: None,
kv_transfer_params: None,
trace_headers: None,
prefill_stats: None,
routed_experts: None,
num_nans_in_logits: 0,
}],
finished_requests: Some(BTreeSet::from(["req-1".to_string()])),
..Default::default()
};
let encoded = encode_msgpack(&outputs).unwrap();
let decoded: EngineCoreOutputs = decode_msgpack(&encoded).unwrap();
assert_eq!(decoded.outputs.len(), 1);
assert_eq!(
decoded.outputs[0].finish_reason,
Some(EngineCoreFinishReason::Length)
);
assert_eq!(
decoded.finished_requests,
Some(BTreeSet::from(["req-1".to_string()]))
);
}
#[test]
fn utility_request_serializes_as_tuple_payload() {
let request = EngineCoreUtilityRequest::new(7, 42, "is_sleeping", ()).unwrap();
let encoded = encode_msgpack(&request).unwrap();
let value = decode_value(&encoded).unwrap();
let array = match value {
Value::Array(array) => array,
other => panic!("expected utility request array, got {other:?}"),
};
assert_eq!(array.len(), 4);
assert_eq!(array[0], Value::from(7));
assert_eq!(array[1], Value::from(42));
assert_eq!(array[2], Value::from("is_sleeping"));
assert_eq!(array[3], Value::Array(Vec::new()));
}
#[test]
fn utility_output_decodes_typed_result() {
let output = UtilityOutput {
call_id: 9,
failure_message: None,
result: Some(utility_result_value(true)),
};
assert!(output.into_typed_result::<bool>("is_sleeping").unwrap());
}
#[test]
fn utility_output_reports_failure_message() {
let error = UtilityOutput {
call_id: 9,
failure_message: Some("boom".to_string()),
result: None,
}
.into_typed_result::<bool>("is_sleeping")
.unwrap_err();
assert!(matches!(
error,
Error::UtilityCallFailed {
method,
call_id,
message
} if method == "is_sleeping" && call_id == 9 && message == "boom"
));
}
#[test]
fn utility_output_decodes_missing_result_as_unit() {
UtilityOutput {
call_id: 3,
failure_message: None,
result: None,
}
.into_typed_result::<()>("reset_mm_cache")
.unwrap();
}
#[test]
fn utility_output_decodes_nil_result_as_unit() {
UtilityOutput {
call_id: 4,
failure_message: None,
result: Some(UtilityResultEnvelope::without_type_info(Value::Nil)),
}
.into_typed_result::<()>("sleep")
.unwrap();
}
#[test]
fn decode_msgpack_includes_type_name_and_value_fallback() {
let error = decode_msgpack::<u64>(
&rmp_serde::to_vec_named(&BTreeMap::from([("status", "READY")])).unwrap(),
)
.unwrap_err();
expect_test::expect![[r#"messagepack decode failed for u64: wrong msgpack marker FixMap(1); value fallback: {"status": "READY"}"#]].assert_eq(&error.to_report_string());
}
}
@@ -0,0 +1,282 @@
use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
use super::tensor::WireTensor;
/// Multimodal feature payload accepted from higher-level frontend code.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/v1/engine/__init__.py#L88>
pub type MmFeatures = Vec<MmFeatureSpec>;
/// Represents a single multimodal input with its processed data and metadata.
///
/// Used to track multimodal data through processing and caching. A request
/// containing multiple multimodal items will have one `MmFeatureSpec`
/// per item.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/multimodal/inputs.py#L301-L332>
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct MmFeatureSpec {
/// Represents multimodal data for this feature.
///
/// Can be `None` if the item is cached, to skip IPC between API server
/// and engine core processes.
pub data: Option<MmKwargsItem>,
/// The input modality, e.g., `"image"`, `"audio"`, `"video"`.
pub modality: String,
/// The hash for caching encoder outputs (with LoRA prefix if applicable).
pub identifier: String,
/// The location of the `modality` tokens corresponding to this item
/// in the prompt, e.g., `PlaceholderRange(offset=2, length=336)`.
pub mm_position: PlaceholderRange,
/// The hash for caching processor outputs (without LoRA prefix).
#[serde(default)]
pub mm_hash: Option<String>,
}
/// Placeholder location information for multi-modal data.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/multimodal/inputs.py#L118-L145>
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct PlaceholderRange {
/// The start index of the placeholder in the prompt.
pub offset: usize,
/// The length of the placeholder.
pub length: usize,
/// A boolean mask of shape `(length,)` indicating which positions
/// between `offset` and `offset + length` to assign embeddings to.
/// `None` means all positions.
#[serde(default)]
pub is_embed: Option<WireTensor>,
}
/// A dictionary of processed keyword arguments to pass to the model,
/// corresponding to a single item in `MultiModalDataItems`.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/multimodal/inputs.py#L854-L871>
pub type MmKwargsItem = BTreeMap<String, MmFieldElem>;
/// Represents a processed keyword argument to pass to a model for a
/// `MmKwargsItem`.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/multimodal/inputs.py#L348-L369>
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct MmFieldElem {
/// The processed value of this field in `MmKwargsItem`, i.e. the
/// keyword argument value to be passed to the model.
///
/// It may be set to `None` if it is determined that the item is cached
/// in `EngineCore`.
pub data: Option<MmKwargValue>,
/// Defines how to combine this field's processed values with others in
/// order to batch multi-modal items together for model inference.
pub field: MmField,
}
/// Processed multimodal keyword argument value.
///
/// Original Python definition (`NestedTensors`) and wire encoding:
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/multimodal/inputs.py#L218-L226>
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/v1/serial_utils.py#L292-L299>
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/v1/serial_utils.py#L456-L465>
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(untagged)]
pub enum MmKwargValue {
Tensor(WireTensor),
Int(i64),
Float(f64),
List(Vec<MmKwargValue>),
}
/// Defines how to interpret tensor data belonging to a keyword argument for
/// `MultiModalKwargsItems`, and vice versa.
///
/// Original Python definitions and wire encoding:
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/multimodal/inputs.py#L385-L630>
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/v1/serial_utils.py#L301-L310>
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/v1/serial_utils.py#L440-L454>
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(try_from = "MmFieldWire", into = "MmFieldWire")]
pub enum MmField {
Batched(MmBatchedField),
Flat(MmFlatField),
Shared(MmSharedField),
}
/// Info: `MultiModalFieldConfig.batched`.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/multimodal/inputs.py#L385-L502>
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct MmBatchedField {
/// If `True`, then this field is excluded from being moved to the
/// accelerator when multimodal items are grouped and batched.
pub keep_on_cpu: bool,
}
/// Info: `MultiModalFieldConfig.flat` and
/// `MultiModalFieldConfig.flat_from_sizes`.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/multimodal/inputs.py#L385-L397>
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/multimodal/inputs.py#L505-L603>
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct MmFlatField {
/// For each multi-modal item, a slice (`dim=0`) or a tuple of slices
/// (`dim>0`) that is used to extract the data corresponding to it.
pub slices: Vec<MmSlice>,
/// The dimension to extract data, default to 0.
pub dim: i32,
/// If `True`, then this field is excluded from being moved to the
/// accelerator when multimodal items are grouped and batched.
pub keep_on_cpu: bool,
}
/// Info: `MultiModalFieldConfig.shared`.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/multimodal/inputs.py#L385-L397>
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/multimodal/inputs.py#L606-L630>
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct MmSharedField {
pub batch_size: usize,
/// If `True`, then this field is excluded from being moved to the
/// accelerator when multimodal items are grouped and batched.
pub keep_on_cpu: bool,
}
/// Python slice encoded as `(start, stop, step)`.
///
/// Original Python wire encoding:
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/v1/serial_utils.py#L199-L204>
#[derive(Debug, Clone, PartialEq, Eq, Serialize_tuple, Deserialize_tuple)]
pub struct SliceSpec {
pub start: Option<isize>,
pub stop: Option<isize>,
pub step: Option<isize>,
}
/// A single slice or a tuple of slices used by `MmFlatField`.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(untagged)]
pub enum MmSlice {
Slice(SliceSpec),
Slices(Vec<SliceSpec>),
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize_tuple, Deserialize_tuple)]
struct MmFieldWire {
name: String,
inner: MmFieldWireInner,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(untagged)]
enum MmFieldWireInner {
Batched(MmBatchedField),
Flat(MmFlatField),
Shared(MmSharedField),
}
impl TryFrom<MmFieldWire> for MmField {
type Error = String;
fn try_from(value: MmFieldWire) -> Result<Self, Self::Error> {
match (value.name.as_str(), value.inner) {
("batched", MmFieldWireInner::Batched(kwargs)) => Ok(Self::Batched(kwargs)),
("flat", MmFieldWireInner::Flat(kwargs)) => Ok(Self::Flat(kwargs)),
("shared", MmFieldWireInner::Shared(kwargs)) => Ok(Self::Shared(kwargs)),
(name, _) => Err(format!(
"mismatched or unknown multimodal field factory {name:?}"
)),
}
}
}
impl From<MmField> for MmFieldWire {
fn from(value: MmField) -> Self {
match value {
MmField::Batched(kwargs) => Self {
name: "batched".to_string(),
inner: MmFieldWireInner::Batched(kwargs),
},
MmField::Flat(kwargs) => Self {
name: "flat".to_string(),
inner: MmFieldWireInner::Flat(kwargs),
},
MmField::Shared(kwargs) => Self {
name: "shared".to_string(),
inner: MmFieldWireInner::Shared(kwargs),
},
}
}
}
#[cfg(test)]
mod tests {
use std::io::Cursor;
use rmpv::Value;
use super::*;
fn encode_value<T: Serialize>(value: &T) -> Value {
let bytes = rmp_serde::to_vec_named(value).expect("encode value");
rmpv::decode::read_value(&mut Cursor::new(bytes)).expect("decode value")
}
#[test]
fn multimodal_field_serializes_to_python_factory_tuple() {
let field = MmField::Flat(MmFlatField {
slices: vec![MmSlice::Slice(SliceSpec {
start: Some(0),
stop: Some(1200),
step: None,
})],
dim: 0,
keep_on_cpu: false,
});
let value = encode_value(&field);
let Value::Array(items) = value else {
panic!("field should encode as tuple array");
};
assert_eq!(items.len(), 2);
assert_eq!(items[0].as_str(), Some("flat"));
let Value::Map(kwargs) = &items[1] else {
panic!("field kwargs should encode as map");
};
assert!(kwargs.iter().any(|(key, _)| key.as_str() == Some("slices")));
assert!(kwargs.iter().any(|(key, _)| key.as_str() == Some("dim")));
assert!(kwargs.iter().any(|(key, _)| key.as_str() == Some("keep_on_cpu")));
}
#[test]
fn multimodal_field_round_trips_python_factory_tuple() {
let field = MmField::Batched(MmBatchedField { keep_on_cpu: true });
let encoded = rmp_serde::to_vec_named(&field).expect("encode field");
let decoded: MmField = rmp_serde::from_slice(&encoded).expect("decode field");
assert_eq!(decoded, field);
}
}
@@ -0,0 +1,192 @@
use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
use crate::protocol::OpaqueValue;
/// Stores cache hit statistics.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/metrics/stats.py#L18-L32>
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct BaseCacheStats {
/// Whether the cache was reset.
pub reset: bool,
/// The number of requests in this update.
pub requests: u64,
/// The number of queries in these requests.
pub queries: u64,
/// The number of hits in these requests.
pub hits: u64,
}
/// Stores prefix cache hit statistics.
/// - `reset`: Whether `reset_prefix_cache` was invoked.
/// - `queries`: Refers to the number of tokens that were queried.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/metrics/stats.py#L114-L143>
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct PrefixCacheStats {
/// Embedded base cache counters and reset flag.
#[serde(flatten)]
pub base: BaseCacheStats,
/// The number of previously preempted requests in this update.
pub preempted_requests: u64,
/// The `queries` number for preempted requests.
pub preempted_queries: u64,
/// The `hits` number for preempted requests.
pub preempted_hits: u64,
}
/// Single KV cache block eviction sample.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/metrics/stats.py#L161-L167>
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct KvCacheEvictionEvent {
/// Lifetime from allocation to eviction.
pub lifetime_seconds: f64,
/// Idle time observed before eviction.
pub idle_seconds: f64,
/// Time gaps between consecutive accesses before eviction.
pub reuse_gaps_seconds: Vec<f64>,
}
/// Per-step iteration decoding stats from scheduler.
///
/// Each scheduler step, statistics on spec decoding performance are aggregated
/// across requests by the scheduler and returned to the frontend in
/// `EngineCoreOutputs -> SchedulerStats`.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/spec_decode/metrics.py#L16-L44>
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct SpecDecodingStats {
/// Configured speculative token count for this scheduler.
pub num_spec_tokens: u64,
/// Number of drafted speculative decoding attempts.
pub num_drafts: u64,
/// Number of drafted tokens.
pub num_draft_tokens: u64,
/// Number of accepted drafted tokens.
pub num_accepted_tokens: u64,
/// Accepted drafted tokens counted by draft position.
pub num_accepted_tokens_per_pos: Vec<u64>,
}
/// Breakdown of a scheduled prefill computation.
///
/// Python models this as a plain `@dataclass`, so it is serialized by msgspec
/// as a map (named fields) rather than in the array-like form used by
/// `EngineCoreOutput` itself.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/d3af8c18317c0dc008d42e4367fbb9045cfb7bf6/vllm/v1/metrics/stats.py#L242-L273>
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct PrefillStats {
/// Total number of tokens to be prefilled.
#[serde(default)]
pub num_prompt_tokens: u32,
/// Tokens to be prefilled locally (actual compute work).
#[serde(default)]
pub num_computed_tokens: u32,
/// Tokens to be prefilled without actual compute work.
#[serde(default)]
pub num_cached_tokens: u32,
/// Tokens to be prefilled from local prefix cache.
#[serde(default)]
pub num_local_cached_tokens: u32,
/// Tokens to be prefilled from external KV transfer.
#[serde(default)]
pub num_external_cached_tokens: u32,
}
/// Stats for debugging the metrics calculation.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/metrics/perf.py#L46-L55>
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct DebugPerfStats {
/// Time spent calculating these stats.
pub calc_duration: f64,
/// Number of prefill requests included in the sampled batch.
pub num_prefill_requests: u64,
/// Number of decode requests included in the sampled batch.
pub num_decode_requests: u64,
/// Optional execution-context breakdown used for debugging.
pub context_breakdown: Option<BTreeMap<String, u64>>,
/// Optional per-component FLOPs breakdown.
pub num_flops_per_gpu_breakdown: Option<BTreeMap<String, u64>>,
/// Optional per-component memory-read breakdown.
pub num_read_bytes_per_gpu_breakdown: Option<BTreeMap<String, u64>>,
/// Optional per-component memory-write breakdown.
pub num_write_bytes_per_gpu_breakdown: Option<BTreeMap<String, u64>>,
}
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/metrics/perf.py#L58-L63>
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct PerfStats {
/// Estimated floating point operations per GPU.
pub num_flops_per_gpu: u64,
/// Estimated bytes read from memory per GPU.
pub num_read_bytes_per_gpu: u64,
/// Estimated bytes written to memory per GPU.
pub num_write_bytes_per_gpu: u64,
/// Optional debug-only perf derivation details.
pub debug_stats: Option<DebugPerfStats>,
}
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/compilation/cuda_graph.py#L28-L33>
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct CudagraphStat {
/// Number of real tokens in the captured batch before padding.
pub num_unpadded_tokens: u64,
/// Number of padded tokens in the captured batch.
pub num_padded_tokens: u64,
/// Number of padding positions added for capture/runtime shape alignment.
pub num_paddings: u64,
/// Runtime mode string associated with this CUDA graph sample.
pub runtime_mode: String,
}
/// Stats associated with the scheduler.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/906a8c15d0beded2cca8b357877266c631340f74/vllm/v1/metrics/stats.py#L170-L196>
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct SchedulerStats {
/// Number of requests in model execution batches.
pub num_running_reqs: u64,
/// Length of the "waiting" request queue.
pub num_waiting_reqs: u64,
/// Length of the "skipped waiting" queue.
#[serde(default)]
pub num_skipped_waiting_reqs: u64,
/// Internal DP load-balancing step counter.
pub step_counter: u64,
/// Internal DP load-balancing wave number.
pub current_wave: u64,
/// KV-cache usage. `1.0` means 100% usage.
pub kv_cache_usage: f64,
/// Local prefix cache statistics.
pub prefix_cache_stats: PrefixCacheStats,
/// External connector prefix cache statistics, when configured.
pub connector_prefix_cache_stats: Option<PrefixCacheStats>,
/// Sampled KV cache eviction events for residency metrics.
pub kv_cache_eviction_events: Vec<KvCacheEvictionEvent>,
/// Speculative decoding scheduler stats, when enabled.
pub spec_decoding_stats: Option<SpecDecodingStats>,
/// Connector-specific KV transfer stats, kept opaque for now.
pub kv_connector_stats: Option<BTreeMap<String, OpaqueValue>>,
/// Waiting request counts per LoRA adapter.
pub waiting_lora_adapters: BTreeMap<String, u64>,
/// Running request counts per LoRA adapter.
pub running_lora_adapters: BTreeMap<String, u64>,
/// CUDA graph runtime stats when graph metrics are enabled.
pub cudagraph_stats: Option<CudagraphStat>,
/// Estimated MFU/performance stats, when enabled.
pub perf_stats: Option<PerfStats>,
}
@@ -0,0 +1,255 @@
use bytemuck::allocation::pod_collect_to_vec;
use enum_as_inner::EnumAsInner;
use half::{bf16, f16};
use rmpv::Value;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
/// Tensors and ndarrays are encoded with this extension type in Python.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/v1/serial_utils.py#L41-L43>
const CUSTOM_TYPE_RAW_VIEW: i8 = 3;
#[easy_ext::ext(ShapeExt)]
impl [usize] {
/// Returned the total number of elements implied by this shape, or `None`
/// if the product of the dimensions overflows `usize`.
pub fn checked_numel(&self) -> Option<usize> {
self.iter().try_fold(1usize, |acc, dim| acc.checked_mul(*dim))
}
}
/// Python ndarray/tensor wire tuple encoded as `(dtype, shape, data)`.
///
/// This matches the custom msgpack representation built by Python
/// `serial_utils.encode_ndarray` / `encode_tensor`.
///
/// Original Python wire encoders:
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/v1/serial_utils.py#L237-L273>
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/v1/serial_utils.py#L389-L425>
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple)]
pub struct WireNdArray {
pub dtype: String,
pub shape: Vec<usize>,
pub data: WireArrayData,
}
impl WireNdArray {
/// Build a float32 tensor/ndarray backed by native-endian raw-view bytes.
pub fn from_f32(shape: Vec<usize>, data: Vec<f32>) -> Result<Self, String> {
validate_element_count(&shape, data.len())?;
Ok(Self {
dtype: "float32".to_string(),
shape,
data: WireArrayData::RawView(pod_collect_to_vec::<f32, u8>(&data)),
})
}
/// Build a float16 tensor/ndarray backed by native-endian raw-view bytes.
pub fn from_f16(shape: Vec<usize>, data: Vec<f16>) -> Result<Self, String> {
validate_element_count(&shape, data.len())?;
Ok(Self {
dtype: "float16".to_string(),
shape,
data: WireArrayData::RawView(pod_collect_to_vec::<f16, u8>(&data)),
})
}
/// Build a bfloat16 tensor/ndarray backed by native-endian raw-view bytes.
pub fn from_bf16(shape: Vec<usize>, data: Vec<bf16>) -> Result<Self, String> {
validate_element_count(&shape, data.len())?;
Ok(Self {
dtype: "bfloat16".to_string(),
shape,
data: WireArrayData::RawView(pod_collect_to_vec::<bf16, u8>(&data)),
})
}
/// Build an int64 tensor/ndarray backed by native-endian raw-view bytes.
pub fn from_i64(shape: Vec<usize>, data: Vec<i64>) -> Result<Self, String> {
validate_element_count(&shape, data.len())?;
Ok(Self {
dtype: "int64".to_string(),
shape,
data: WireArrayData::RawView(pod_collect_to_vec::<i64, u8>(&data)),
})
}
/// Build a uint32 tensor/ndarray backed by native-endian raw-view bytes.
pub fn from_u32(shape: Vec<usize>, data: Vec<u32>) -> Result<Self, String> {
validate_element_count(&shape, data.len())?;
Ok(Self {
dtype: "uint32".to_string(),
shape,
data: WireArrayData::RawView(pod_collect_to_vec::<u32, u8>(&data)),
})
}
/// Build a bool tensor/ndarray backed by raw-view bytes.
///
/// This matches `torch.bool` storage: one byte per element, not a packed
/// bitmap. Values are canonicalized as `false -> 0` and `true -> 1`.
pub fn from_bool(shape: Vec<usize>, data: Vec<bool>) -> Result<Self, String> {
validate_element_count(&shape, data.len())?;
Ok(Self {
dtype: "bool".to_string(),
shape,
data: WireArrayData::RawView(data.iter().map(|value| u8::from(*value)).collect()),
})
}
/// Build a tensor/ndarray from already-encoded raw-view bytes.
///
/// Use this as an escape hatch when the caller already owns bytes that
/// match the requested `dtype` and `shape`.
pub fn from_raw(dtype: impl Into<String>, shape: Vec<usize>, data: Vec<u8>) -> Self {
Self {
dtype: dtype.into(),
shape,
data: WireArrayData::RawView(data),
}
}
}
/// Validate that the number of elements implied by the shape matches the length
/// of the data.
fn validate_element_count(shape: &[usize], len: usize) -> Result<(), String> {
let expected = shape
.checked_numel()
.ok_or_else(|| format!("tensor shape product overflows usize: {shape:?}"))?;
if expected == len {
Ok(())
} else {
Err(format!(
"tensor data length {len} does not match shape {shape:?} product {expected}"
))
}
}
/// Python tensor wire tuple encoded as `(dtype, shape, data)`.
///
/// This is the same wire shape as [`WireNdArray`]; multimodal request payloads
/// use it for `torch.Tensor` values.
pub type WireTensor = WireNdArray;
/// Python array/tensor payload reference inside [`WireNdArray`].
///
/// The data can be either an inline msgpack raw-view extension or an index into
/// the multipart aux-frame list carried alongside the primary msgpack frame.
///
/// Original Python wire encoders:
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/v1/serial_utils.py#L237-L273>
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/v1/serial_utils.py#L389-L425>
#[derive(Debug, Clone, PartialEq, EnumAsInner)]
pub enum WireArrayData {
/// The index of the aux frame where the raw bytes of this array/tensor are
/// stored.
AuxIndex(usize),
/// The raw bytes of this array/tensor.
RawView(Vec<u8>),
}
impl<'de> Deserialize<'de> for WireArrayData {
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let value = Value::deserialize(deserializer)?;
match value {
Value::Ext(tag, bytes) if tag == CUSTOM_TYPE_RAW_VIEW => Ok(Self::RawView(bytes)),
Value::Ext(tag, _) => Err(serde::de::Error::custom(format!(
"unsupported extension type code {tag}"
))),
Value::Integer(index) => {
index.as_u64().map(|index| Self::AuxIndex(index as usize)).ok_or_else(|| {
serde::de::Error::custom("aux frame index must be a non-negative integer")
})
}
other => Err(serde::de::Error::custom(format!(
"expected raw-view ext or aux frame index, got {other:?}"
))),
}
}
}
impl Serialize for WireArrayData {
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
where
S: Serializer,
{
// TODO: outbound request serialization currently only supports inline
// raw-view bytes. Emitting aux frames needs transport-level plumbing;
// serializing `AuxIndex` here only preserves an already-built reference.
match self {
Self::AuxIndex(index) => serializer.serialize_u64(*index as u64),
Self::RawView(bytes) => {
Value::Ext(CUSTOM_TYPE_RAW_VIEW, bytes.clone()).serialize(serializer)
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn constructors_build_raw_view_tensors() {
let f32_tensor = WireNdArray::from_f32(vec![2], vec![1.0, 2.5]).unwrap();
assert_eq!(f32_tensor.dtype, "float32");
assert_eq!(f32_tensor.shape, vec![2]);
assert_eq!(
f32_tensor.data.into_raw_view().expect("raw view"),
[1.0_f32, 2.5].into_iter().flat_map(f32::to_ne_bytes).collect::<Vec<_>>()
);
let f16_tensor =
WireNdArray::from_f16(vec![2], vec![f16::from_f32(1.0), f16::from_f32(2.5)]).unwrap();
assert_eq!(f16_tensor.dtype, "float16");
assert_eq!(f16_tensor.shape, vec![2]);
assert_eq!(f16_tensor.data.into_raw_view().expect("raw view").len(), 4);
let bf16_tensor =
WireNdArray::from_bf16(vec![2], vec![bf16::from_f32(1.0), bf16::from_f32(2.5)])
.unwrap();
assert_eq!(bf16_tensor.dtype, "bfloat16");
assert_eq!(bf16_tensor.shape, vec![2]);
assert_eq!(bf16_tensor.data.into_raw_view().expect("raw view").len(), 4);
let i64_tensor = WireNdArray::from_i64(vec![1], vec![-7]).unwrap();
assert_eq!(i64_tensor.dtype, "int64");
assert_eq!(
i64_tensor.data.into_raw_view().expect("raw view"),
(-7_i64).to_ne_bytes()
);
let u32_tensor = WireNdArray::from_u32(vec![1], vec![42]).unwrap();
assert_eq!(u32_tensor.dtype, "uint32");
assert_eq!(
u32_tensor.data.into_raw_view().expect("raw view"),
42_u32.to_ne_bytes()
);
let bool_tensor = WireNdArray::from_bool(vec![2], vec![false, true]).unwrap();
assert_eq!(bool_tensor.dtype, "bool");
assert_eq!(
bool_tensor.data.into_raw_view().expect("raw view"),
vec![0, 1]
);
let raw_tensor = WireNdArray::from_raw("custom", vec![3], vec![1, 2, 3]);
assert_eq!(raw_tensor.dtype, "custom");
assert_eq!(raw_tensor.shape, vec![3]);
assert_eq!(
raw_tensor.data.into_raw_view().expect("raw view"),
vec![1, 2, 3]
);
}
#[test]
fn constructors_validate_shape_product() {
let err = WireNdArray::from_f32(vec![2, 2], vec![1.0, 2.0]).unwrap_err();
assert!(err.contains("does not match shape"));
}
}
@@ -0,0 +1,295 @@
use std::future::Future;
use std::path::Path;
use std::pin::Pin;
use std::time::Duration;
use tempfile::TempDir;
use tokio::sync::oneshot;
use zeromq::prelude::{Socket, SocketRecv, SocketSend};
use zeromq::util::PeerIdentity;
use zeromq::{DealerSocket, PushSocket, SocketOptions, SubSocket, ZmqMessage};
use crate::EngineId;
use crate::protocol::ModelDtype;
use crate::protocol::handshake::{EngineCoreReadyResponse, HandshakeInitMessage, ReadyMessage};
/// Per-test IPC endpoint namespace backed by a unique temporary directory.
///
/// Using one directory per test avoids endpoint collisions without requiring
/// ad-hoc unique-name generation at each call site.
#[derive(Debug)]
pub struct IpcNamespace {
dir: TempDir,
}
impl IpcNamespace {
/// Create a fresh namespace for one test case.
pub fn new() -> std::io::Result<Self> {
Ok(Self {
dir: TempDir::new()?,
})
}
/// Build one `ipc://...` endpoint under this namespace.
pub fn endpoint(&self, name: impl AsRef<Path>) -> String {
let path = self.dir.path().join(name);
format!("ipc://{}", path.to_string_lossy())
}
/// Endpoint used for the initial READY/HELLO handshake.
pub fn handshake_endpoint(&self) -> String {
self.endpoint("handshake.sock")
}
/// Endpoint used for engine-core request traffic.
pub fn input_endpoint(&self) -> String {
self.endpoint("input.sock")
}
/// Endpoint used for engine-core output traffic.
pub fn output_endpoint(&self) -> String {
self.endpoint("output.sock")
}
}
/// Construct a standard local READY message used by mock engines in tests.
fn ready_message(status: &str) -> ReadyMessage {
ReadyMessage {
status: Some(status.to_string()),
local: Some(true),
headless: Some(true),
parallel_config_hash: None,
}
}
/// Construct a default ready response payload for mock engine input
/// registration.
fn ready_response_payload() -> Vec<u8> {
rmp_serde::to_vec_named(&EngineCoreReadyResponse {
max_model_len: 4096,
num_gpu_blocks: 0,
dp_stats_address: None,
dtype: Some(ModelDtype::Float32),
})
.expect("encode ready response payload")
}
/// Coordinator-side sockets connected by one mock engine when coordinator mode
/// is enabled.
pub struct MockCoordinatorConnections {
/// Subscription socket that receives coordinator broadcasts such as
/// `START_DP_WAVE`.
pub input_sub: SubSocket,
/// Push socket used to send coordinator-only `EngineCoreOutputs` back to
/// the frontend.
pub output_push: PushSocket,
}
/// Fully connected mock engine transport state used by tests.
pub struct MockEngineConnections {
/// Decoded INIT message sent by the frontend during handshake.
pub init: HandshakeInitMessage,
/// Socket used to receive frontend requests.
pub dealer: DealerSocket,
/// Socket used to publish normal request outputs back to the frontend.
pub push: PushSocket,
/// Optional coordinator sockets when the client enabled the in-process
/// coordinator.
pub coordinator: Option<MockCoordinatorConnections>,
}
/// Complete the engine-core handshake and connect mock input/output sockets
/// plus optional coordinator sockets.
pub async fn setup_mock_engine_connections(
engine_handshake: String,
engine_id: impl Into<EngineId>,
) -> MockEngineConnections {
// Wait for the client to bind the handshake socket before connecting.
// A fixed sleep is racy under CI load; instead poll for the socket file.
let socket_path = engine_handshake
.strip_prefix("ipc://")
.expect("handshake address must be ipc://");
for _ in 0..100 {
if Path::new(socket_path).exists() {
break;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
let peer_identity = PeerIdentity::try_from(engine_id.into()).expect("peer id");
let mut options = SocketOptions::default();
options.peer_identity(peer_identity.clone());
let mut handshake = DealerSocket::with_options(options);
handshake
.connect(&engine_handshake)
.await
.expect("connect mock engine handshake socket");
handshake
.send(ZmqMessage::from(
rmp_serde::to_vec_named(&ready_message("HELLO")).expect("encode HELLO ready message"),
))
.await
.expect("send HELLO ready message");
let init_frames = handshake.recv().await.expect("receive handshake init message").into_vec();
assert_eq!(init_frames.len(), 1);
let init: HandshakeInitMessage =
rmp_serde::from_slice(init_frames[0].as_ref()).expect("decode handshake init message");
let mut input_options = SocketOptions::default();
input_options.peer_identity(peer_identity);
let mut dealer = DealerSocket::with_options(input_options);
dealer
.connect(&init.addresses.inputs[0])
.await
.expect("connect mock engine input socket");
dealer
.send(ZmqMessage::from(ready_response_payload()))
.await
.expect("send mock engine input ready frame");
let mut push = PushSocket::new();
push.connect(&init.addresses.outputs[0])
.await
.expect("connect mock engine output socket");
let coordinator = match (
init.addresses.coordinator_input.as_deref(),
init.addresses.coordinator_output.as_deref(),
) {
(Some(coordinator_input), Some(coordinator_output)) => {
let mut input_sub = SubSocket::new();
input_sub
.connect(coordinator_input)
.await
.expect("connect mock engine coordinator input socket");
input_sub
.subscribe("")
.await
.expect("subscribe mock engine coordinator input socket");
let mut output_push = PushSocket::new();
output_push
.connect(coordinator_output)
.await
.expect("connect mock engine coordinator output socket");
let ready =
input_sub.recv().await.expect("receive coordinator READY marker").into_vec();
assert_eq!(ready.len(), 1);
assert_eq!(ready[0].as_ref(), b"READY");
Some(MockCoordinatorConnections {
input_sub,
output_push,
})
}
(None, None) => None,
_ => panic!("coordinator handshake addresses must be both present or both absent"),
};
handshake
.send(ZmqMessage::from(
rmp_serde::to_vec_named(&ready_message("READY")).expect("encode READY ready message"),
))
.await
.expect("send READY ready message");
MockEngineConnections {
init,
dealer,
push,
coordinator,
}
}
/// Connect one mock engine directly to already-bootstrapped frontend
/// input/output sockets.
pub async fn setup_bootstrapped_mock_engine(
input_address: String,
output_address: String,
engine_id: impl Into<EngineId>,
) -> (DealerSocket, PushSocket) {
for endpoint in [&input_address, &output_address] {
if let Some(socket_path) = endpoint.strip_prefix("ipc://") {
for _ in 0..100 {
if Path::new(socket_path).exists() {
break;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
}
}
let peer_identity = PeerIdentity::try_from(engine_id.into()).expect("peer id");
let mut input_options = SocketOptions::default();
input_options.peer_identity(peer_identity);
let mut dealer = DealerSocket::with_options(input_options);
dealer.connect(&input_address).await.expect("connect mock engine input socket");
dealer
.send(ZmqMessage::from(ready_response_payload()))
.await
.expect("send mock engine input ready frame");
let mut push = PushSocket::new();
push.connect(&output_address).await.expect("connect mock engine output socket");
(dealer, push)
}
/// Complete the engine-core handshake and connect mock input/output sockets.
///
/// This returns the decoded handshake init message plus the `DealerSocket` used
/// to receive client requests and the `PushSocket` used to send engine outputs
/// back to the client.
pub async fn setup_mock_engine_with_init(
engine_handshake: String,
engine_id: impl Into<EngineId>,
) -> (HandshakeInitMessage, DealerSocket, PushSocket) {
let MockEngineConnections {
init, dealer, push, ..
} = setup_mock_engine_connections(engine_handshake, engine_id).await;
(init, dealer, push)
}
/// Complete the engine-core handshake and connect mock input/output sockets.
///
/// This returns the `DealerSocket` used to receive client requests and the
/// `PushSocket` used to send engine outputs back to the client.
pub async fn setup_mock_engine(
engine_handshake: String,
engine_id: impl Into<EngineId>,
) -> (DealerSocket, PushSocket) {
let (_, dealer, push) = setup_mock_engine_with_init(engine_handshake, engine_id).await;
(dealer, push)
}
/// Spawn a mock engine task and keep its sockets alive until the returned
/// shutdown sender is triggered by the test.
///
/// The script borrows the connected sockets mutably while it runs. After the
/// script completes, this helper keeps the sockets alive until the test
/// explicitly signals shutdown.
pub fn spawn_mock_engine_task<F>(
engine_handshake: String,
engine_id: impl Into<EngineId>,
run: F,
) -> (oneshot::Sender<()>, tokio::task::JoinHandle<()>)
where
F: for<'a> FnOnce(
&'a mut DealerSocket,
&'a mut PushSocket,
) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>>
+ Send
+ 'static,
{
let (shutdown_tx, shutdown_rx) = oneshot::channel();
let engine_id = engine_id.into();
let engine_task = tokio::spawn(async move {
let (mut dealer, mut push) = setup_mock_engine(engine_handshake, engine_id).await;
run(&mut dealer, &mut push).await;
let _ = shutdown_rx.await;
});
(shutdown_tx, engine_task)
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1 @@
mod client;
+356
View File
@@ -0,0 +1,356 @@
#!/usr/bin/env -S uv run
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# /// script
# requires-python = ">=3.11"
# dependencies = [
# "msgspec>=0.19,<1",
# "msgpack>=1,<2",
# "numpy>=2,<3",
# ]
# ///
from enum import Enum, IntEnum
import msgpack
import msgspec
import numpy as np
class RequestOutputKind(Enum):
DELTA = 1
FINAL_ONLY = 2
class FinishReason(IntEnum):
STOP = 0
LENGTH = 1
ABORT = 2
ERROR = 3
REPETITION = 4
class EngineCoreSamplingParams(msgspec.Struct, dict=True):
temperature: float = 1.0
top_p: float = 1.0
top_k: int = 0
seed: int | None = None
max_tokens: int = 65536
min_tokens: int = 0
min_p: float = 0.0
frequency_penalty: float = 0.0
presence_penalty: float = 0.0
repetition_penalty: float = 1.0
stop_token_ids: list[int] = []
_eos_token_id: int | None = None
_all_stop_token_ids: set[int] = set()
output_kind: RequestOutputKind = RequestOutputKind.DELTA
class EngineCoreRequest(
msgspec.Struct,
array_like=True,
omit_defaults=True,
):
request_id: str
prompt_token_ids: list[int] | None
mm_features: object | None
sampling_params: EngineCoreSamplingParams | None
pooling_params: object | None
arrival_time: float
lora_request: object | None = None
cache_salt: str | None = None
data_parallel_rank: int | None = None
prompt_embeds: object | None = None
prompt_is_token_ids: list[bool] | None = None
client_index: int = 0
current_wave: int = 0
priority: int = 0
trace_headers: dict[str, str] | None = None
resumable: bool = False
external_req_id: str | None = None
reasoning_ended: bool | None = None
reasoning_parser_kwargs: dict[str, object] | None = None
abort_immediately: bool = False
class EngineCoreOutput(
msgspec.Struct,
array_like=True,
omit_defaults=True,
):
request_id: str
new_token_ids: list[int]
new_logprobs: object | None = None
new_prompt_logprobs_tensors: object | None = None
pooling_output: object | None = None
finish_reason: FinishReason | None = None
stop_reason: int | str | None = None
events: object | None = None
kv_transfer_params: object | None = None
trace_headers: object | None = None
prefill_stats: object | None = None
routed_experts: object | None = None
num_nans_in_logits: int = 0
class EngineCoreOutputs(
msgspec.Struct,
array_like=True,
omit_defaults=True,
):
engine_index: int = 0
outputs: list[EngineCoreOutput] = []
scheduler_stats: object | None = None
timestamp: float = 0.0
utility_output: object | None = None
finished_requests: set[str] | None = None
wave_complete: int | None = None
start_wave: int | None = None
request = EngineCoreRequest(
request_id="req-1",
prompt_token_ids=[11, 22],
mm_features=None,
sampling_params=EngineCoreSamplingParams(
temperature=0.8,
top_p=0.9,
top_k=8,
seed=None,
max_tokens=32,
min_tokens=1,
min_p=0.0,
frequency_penalty=0.0,
presence_penalty=0.0,
repetition_penalty=1.0,
stop_token_ids=[151643],
_eos_token_id=151645,
_all_stop_token_ids={151643, 151645},
output_kind=RequestOutputKind.FINAL_ONLY,
),
pooling_params=None,
arrival_time=42.5,
client_index=0,
)
multimodal_tensor = np.array([[1.0, 2.0], [3.5, 4.25]], dtype=np.float32)
multimodal_features = [
{
"data": {
"pixel_values": {
"data": [
"float32",
[2, 2],
msgpack.ExtType(3, multimodal_tensor.tobytes()),
],
"field": [
"flat",
{
"slices": [[0, 2, None]],
"dim": 0,
"keep_on_cpu": False,
},
],
}
},
"modality": "image",
"identifier": "mm-cache-key",
"mm_position": {
"offset": 1,
"length": 2,
"is_embed": None,
},
"mm_hash": "processor-hash",
}
]
multimodal_request_wire = [
"req-mm",
[101, 102, 103, 104],
multimodal_features,
None,
None,
43.5,
]
outputs = EngineCoreOutputs(
outputs=[
EngineCoreOutput(
request_id="req-1",
new_token_ids=[7, 8],
finish_reason=FinishReason.LENGTH,
)
],
finished_requests={"req-1"},
)
def encode_ndarray(
array: np.ndarray,
buffers: list[bytes],
*,
size_threshold: int = 256,
):
arr_data = array.data if array.flags.c_contiguous else array.tobytes()
if not array.shape or array.nbytes < size_threshold:
data = msgpack.ExtType(3, bytes(arr_data))
else:
data = len(buffers)
buffers.append(bytes(arr_data))
return [array.dtype.str, list(array.shape), data]
def encode_tensor_like(
dtype: str,
shape: list[int],
payload: bytes,
buffers: list[bytes],
*,
size_threshold: int = 256,
):
if len(payload) < size_threshold:
data = msgpack.ExtType(3, payload)
else:
data = len(buffers)
buffers.append(payload)
return [dtype, shape, data]
def encode_output_frames(obj, *, size_threshold: int = 256) -> list[bytes]:
buffers = [b""]
def transform(value):
if isinstance(value, np.ndarray):
return encode_ndarray(value, buffers, size_threshold=size_threshold)
if (
isinstance(value, tuple)
and len(value) == 3
and value[0] in ("int32", "int64", "float32")
):
dtype, shape, payload = value
return encode_tensor_like(
dtype,
shape,
payload,
buffers,
size_threshold=size_threshold,
)
if type(value) is list:
return [transform(v) for v in value]
if type(value) is tuple:
return [transform(v) for v in value]
if type(value) is dict:
return {k: transform(v) for k, v in value.items()}
return value
buffers[0] = msgpack.packb(transform(obj), use_bin_type=True)
return buffers
def engine_output_wire(
request_id: str,
*,
new_logprobs=None,
new_prompt_logprobs_tensors=None,
):
return [
request_id,
[7, 8],
new_logprobs,
new_prompt_logprobs_tensors,
None,
int(FinishReason.LENGTH),
]
def engine_outputs_wire(output):
return [0, [output], None, 0.0, None, ["req-1"]]
inline_logprobs = engine_outputs_wire(
engine_output_wire(
"req-1",
new_logprobs=(
np.array([[1, 2, 3], [4, 5, 6]], dtype=np.int64),
np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float32),
np.array([1, 2], dtype=np.int64),
None,
),
)
)
multipart_logprobs = engine_outputs_wire(
engine_output_wire(
"req-1",
new_logprobs=(
np.array([[1, 2, 3], [4, 5, 6]], dtype=np.int64),
np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float32),
np.array([1, 2], dtype=np.int64),
None,
),
)
)
inline_prompt_logprobs = engine_outputs_wire(
engine_output_wire(
"req-1",
new_prompt_logprobs_tensors=(
(
"int64",
[2, 3],
np.array([[10, 11, 12], [13, 14, 15]], dtype=np.int64).tobytes(),
),
(
"float32",
[2, 3],
np.array(
[[10, 11, 12], [13, 14, 15]],
dtype=np.float32,
).tobytes(),
),
("int64", [2], np.array([3, 4], dtype=np.int64).tobytes()),
None,
),
)
)
multipart_prompt_logprobs = engine_outputs_wire(
engine_output_wire(
"req-1",
new_prompt_logprobs_tensors=(
(
"int64",
[2, 3],
np.array([[10, 11, 12], [13, 14, 15]], dtype=np.int64).tobytes(),
),
(
"float32",
[2, 3],
np.array(
[[10, 11, 12], [13, 14, 15]],
dtype=np.float32,
).tobytes(),
),
("int64", [2], np.array([3, 4], dtype=np.int64).tobytes()),
None,
),
)
)
print(msgspec.msgpack.encode(request).hex())
print(msgpack.packb(multimodal_request_wire, use_bin_type=True).hex())
print(msgspec.msgpack.encode(outputs).hex())
print(" ".join(frame.hex() for frame in encode_output_frames(inline_logprobs)))
print(
" ".join(
frame.hex()
for frame in encode_output_frames(multipart_logprobs, size_threshold=1)
)
)
print(" ".join(frame.hex() for frame in encode_output_frames(inline_prompt_logprobs)))
print(
" ".join(
frame.hex()
for frame in encode_output_frames(multipart_prompt_logprobs, size_threshold=1)
)
)
@@ -0,0 +1,606 @@
use std::collections::{BTreeMap, BTreeSet};
use std::fmt::Debug;
use std::ops::Deref;
use std::time::Duration;
use bytes::Bytes;
use enum_as_inner::EnumAsInner;
use thiserror_ext::AsReport;
use tokio::sync::mpsc;
use tokio::time::timeout;
use tracing::{debug, error, info, trace, warn};
use zeromq::prelude::{Socket, SocketRecv, SocketSend};
use zeromq::util::PeerIdentity;
use zeromq::{PullSocket, RouterSendHalf, RouterSocket, ZmqError, ZmqMessage};
use crate::coordinator::CoordinatorBootstrap;
use crate::error::{Error, Result, bail_unexpected_handshake_message};
use crate::protocol::handshake::{
EngineCoreReadyResponse, HandshakeAddresses, HandshakeInitMessage, ReadyMessage,
};
use crate::protocol::{
EngineCoreOutputs, decode_engine_core_outputs, decode_msgpack, encode_msgpack,
};
/// Dedicated single-frame sentinel emitted by Python `EngineCoreProc` when the
/// engine dies.
pub const ENGINE_CORE_DEAD_SENTINEL: &[u8] = b"ENGINE_CORE_DEAD";
/// Opaque routing identity of one engine on the frontend transport.
#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct EngineId(Bytes);
impl Debug for EngineId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
// Display the engine id as a hex string for easier debugging.
write!(f, "EngineId({})", hex::encode(&self.0))
}
}
impl EngineId {
/// Convert the engine id into a ZMQ frame for sending.
pub fn to_frame(&self) -> Bytes {
self.0.clone()
}
/// Convert the engine id into a ZMQ frame for sending.
pub fn into_frame(self) -> Bytes {
self.0
}
/// Parse the Python-compatible engine index encoded in the routing
/// identity.
///
/// Python `EngineCoreProc` currently uses a two-byte little-endian engine
/// index as its ROUTER/DEALER identity. Coordinator control messages
/// such as `START_DP_WAVE(exclude_engine_index)` need that engine-side
/// index rather than any frontend-local ordering.
pub fn engine_index(&self) -> Option<u32> {
if self.len() != 2 {
return None;
}
Some(u16::from_le_bytes([self[0], self[1]]) as u32)
}
/// Construct an engine id from the Python-compatible engine index encoding
/// (two-byte little-endian).
pub fn from_engine_index(value: u32) -> Self {
Self(Bytes::copy_from_slice(&(value as u16).to_le_bytes()))
}
}
impl Deref for EngineId {
type Target = [u8];
fn deref(&self) -> &Self::Target {
self.0.as_ref()
}
}
impl From<Vec<u8>> for EngineId {
fn from(value: Vec<u8>) -> Self {
Self(Bytes::from(value))
}
}
impl<const N: usize> From<&[u8; N]> for EngineId {
fn from(value: &[u8; N]) -> Self {
Self(Bytes::copy_from_slice(value))
}
}
impl TryFrom<EngineId> for PeerIdentity {
type Error = ZmqError;
fn try_from(value: EngineId) -> std::result::Result<Self, Self::Error> {
PeerIdentity::try_from(value.into_frame())
}
}
/// Per-engine handshake result collected while bootstrapping one shared
/// transport.
#[derive(Clone, Debug)]
pub struct ConnectedEngine {
/// The identity of the connected engine.
pub engine_id: EngineId,
/// Post-initialization configuration received from the engine on the input
/// socket registration message. `None` until the registration is received.
pub ready_response: Option<EngineCoreReadyResponse>,
}
/// Represents the connected shared transport plus all registered engines after
/// a successful multi-engine startup handshake.
pub struct ConnectedTransport {
/// The local address of the shared input socket that all engines connect to
/// for receiving requests.
pub input_address: String,
/// The local address of the shared output socket that all engines connect
/// to for sending responses.
pub output_address: String,
/// All engines connected through the startup handshake.
pub engines: Vec<ConnectedEngine>,
/// Optional engine-facing coordinator transport used for in-process wave
/// coordination.
pub coordinator: Option<CoordinatorBootstrap>,
/// The sending half of the shared input socket.
pub input_send: RouterSendHalf,
/// The shared output socket for receiving responses from all engines.
pub output_socket: PullSocket,
}
#[derive(Clone, Debug, EnumAsInner)]
enum EngineStartupState {
HelloReceived,
ReadyReceived,
}
/// Connect to one or more engines through the startup handshake protocol,
/// returning the shared data-plane transport plus the registered engines.
pub async fn connect_handshake(
handshake_address: &str,
engine_count: usize,
local_host: &str,
local_input_address: Option<&str>,
local_output_address: Option<&str>,
enable_inproc_coordinator: bool,
ready_timeout: Duration,
) -> Result<ConnectedTransport> {
if engine_count == 0 {
bail_unexpected_handshake_message!("expected engine_count >= 1");
}
info!(
engine_count,
handshake_address, "waiting for engines to connect"
);
// 1. Bind shared local input/output sockets first so every engine receives the same data-plane
// addresses during handshake.
debug!(
local_host,
?ready_timeout,
engine_count,
"binding shared transport sockets"
);
let (input_address, mut input_socket, output_address, output_socket) =
bind_local_sockets(local_host, local_input_address, local_output_address).await?;
info!(%input_address, %output_address, "bound local transport sockets");
let mut coordinator = if enable_inproc_coordinator {
Some(CoordinatorBootstrap::bind(local_host).await?)
} else {
None
};
// 2. Bind the shared handshake socket once. All engines connect to this socket with their own
// identities, and startup order does not matter.
let mut handshake_socket = RouterSocket::new();
handshake_socket.bind(handshake_address).await?;
let mut engines = BTreeMap::new();
// 3. Receive HELLO from every engine and send a matching INIT. When coordinator mode is
// enabled, the engines will not emit READY until the coordinator barrier below completes.
while engines.len() < engine_count {
debug!(
handshake_address,
connected = engines.len(),
waiting_for = engine_count,
"waiting for engine HELLO"
);
let message = timeout(ready_timeout, handshake_socket.recv()).await.map_err(|_| {
Error::HandshakeTimeout {
stage: "HELLO",
timeout: ready_timeout,
}
})??;
let (engine_id, handshake_message) = decode_handshake_message(message, None)?;
match handshake_message.status.as_deref() {
Some("HELLO") => {
if engines.contains_key(&engine_id) {
bail_unexpected_handshake_message!(
"duplicate engine id {engine_id:?} observed during startup handshake"
);
}
debug!(handshake_address, ?engine_id, "received HELLO from engine");
send_init_message(
&mut handshake_socket,
&engine_id,
&input_address,
&output_address,
coordinator.as_ref(),
)
.await?;
debug!(handshake_address, ?engine_id, "sent INIT to engine");
engines.insert(engine_id.clone(), EngineStartupState::HelloReceived);
}
Some("READY") => {
if coordinator.is_some() {
bail_unexpected_handshake_message!(
"received READY for engine id {engine_id:?} before coordinator startup gate completed"
);
}
let state = match engines.get_mut(&engine_id) {
Some(state) if !state.is_ready_received() => state,
_ => {
bail_unexpected_handshake_message!(
"received READY for unexpected or duplicate engine id {engine_id:?}"
);
}
};
debug!(
handshake_address,
?engine_id,
?handshake_message,
"received overlapping READY from engine during HELLO phase"
);
*state = EngineStartupState::ReadyReceived;
}
other => {
bail_unexpected_handshake_message!("unexpected handshake status {other:?}");
}
}
}
// 4. Optional coordinator startup gate. Without coordinator there is nothing to do.
if let Some(coordinator) = coordinator.as_mut() {
coordinator.wait_for_startup_gate(engine_count, ready_timeout).await?;
}
// 5. After the optional gate has opened, every engine may now send READY.
while engines.values().any(|state| !state.is_ready_received()) {
debug!(
handshake_address,
connected = engines.len(),
ready = engines.values().filter(|state| state.is_ready_received()).count(),
waiting_for = engine_count,
"waiting for engine READY"
);
let message = timeout(ready_timeout, handshake_socket.recv()).await.map_err(|_| {
Error::HandshakeTimeout {
stage: "READY",
timeout: ready_timeout,
}
})??;
let (engine_id, handshake_message) = decode_handshake_message(message, None)?;
match handshake_message.status.as_deref() {
Some("READY") => {
let state = match engines.get_mut(&engine_id) {
Some(state) if !state.is_ready_received() => state,
_ => {
bail_unexpected_handshake_message!(
"received READY for unexpected or duplicate engine id {engine_id:?}"
);
}
};
debug!(
handshake_address,
?engine_id,
?handshake_message,
"received READY from engine"
);
*state = EngineStartupState::ReadyReceived;
}
Some("HELLO") => {
bail_unexpected_handshake_message!(
"received duplicate HELLO for engine id {engine_id:?} after INIT phase completed"
);
}
other => {
bail_unexpected_handshake_message!("unexpected handshake status {other:?}");
}
}
}
// 4. Wait for every engine to connect to the shared input socket and register itself. The
// `ready_response` is a placeholder; it is populated for each engine by
// `wait_for_input_registrations` below.
let mut engines: Vec<_> = engines
.into_keys()
.map(|engine_id| ConnectedEngine {
engine_id,
ready_response: None,
})
.collect();
wait_for_input_registrations(&mut input_socket, &mut engines, ready_timeout).await?;
debug!(
engine_count = engines.len(),
"all engines registered on shared input socket"
);
info!(engine_count = engines.len(), "engines connected");
let (input_send, _) = input_socket.split();
Ok(ConnectedTransport {
input_address,
output_address,
input_send,
output_socket,
engines,
coordinator,
})
}
/// Bind to Python-supplied frontend transport addresses and wait for
/// already-initialized engines to register themselves on the input socket.
///
/// This path mirrors Python's externally managed `AsyncMPClient` bootstrap
/// model: the addresses are already fixed by the supervisor, and engine
/// identities are synthesized from contiguous rank order instead of being
/// discovered through a Rust-owned handshake.
pub async fn connect_bootstrapped(
input_address: &str,
output_address: &str,
engine_count: usize,
ready_timeout: Duration,
) -> Result<ConnectedTransport> {
if engine_count == 0 {
bail_unexpected_handshake_message!("expected engine_count >= 1");
}
let mut input_socket = RouterSocket::new();
let input_address = input_socket.bind(input_address).await?.to_string();
let mut output_socket = PullSocket::new();
let output_address = output_socket.bind(output_address).await?.to_string();
// TODO: follow start rank
let mut engines = (0..engine_count)
.map(|index| ConnectedEngine {
engine_id: EngineId::from((index as u16).to_le_bytes().to_vec()),
ready_response: None,
})
.collect::<Vec<_>>();
wait_for_input_registrations(&mut input_socket, &mut engines, ready_timeout).await?;
info!(
engine_count = engines.len(),
"bootstrapped engines connected"
);
let (input_send, _) = input_socket.split();
Ok(ConnectedTransport {
input_address,
output_address,
engines,
coordinator: None,
input_send,
output_socket,
})
}
/// Bind new input and output sockets.
async fn bind_local_sockets(
local_host: &str,
local_input_address: Option<&str>,
local_output_address: Option<&str>,
) -> Result<(String, RouterSocket, String, PullSocket)> {
let mut input_socket = RouterSocket::new();
let input_bind_address = local_input_address
.map(str::to_owned)
.unwrap_or_else(|| format!("tcp://{local_host}:0"));
let input_address = input_socket.bind(&input_bind_address).await?.to_string();
let mut output_socket = PullSocket::new();
let output_bind_address = local_output_address
.map(str::to_owned)
.unwrap_or_else(|| format!("tcp://{local_host}:0"));
let output_address = output_socket.bind(&output_bind_address).await?.to_string();
Ok((input_address, input_socket, output_address, output_socket))
}
/// Decode a handshake message and validate its structure and identity.
fn decode_handshake_message(
message: ZmqMessage,
expected_id: Option<&EngineId>,
) -> Result<(EngineId, ReadyMessage)> {
if message.len() != 2 {
bail_unexpected_handshake_message!("expected 2 frames, got {}", message.len());
}
let frames = message.into_vec();
let actual_id = EngineId(frames[0].clone());
if let Some(expected_id) = expected_id
&& actual_id != *expected_id
{
return Err(Error::UnexpectedHandshakeIdentity {
expected: expected_id.to_vec(),
actual: actual_id.to_vec(),
});
}
let handshake_message: ReadyMessage = decode_msgpack(&frames[1])?;
Ok((actual_id, handshake_message))
}
/// Send an INIT message to the engine with the local socket addresses for the
/// engine to connect to, using the handshake socket.
async fn send_init_message(
handshake_socket: &mut RouterSocket,
engine_id: &EngineId,
input_address: &str,
output_address: &str,
coordinator: Option<&CoordinatorBootstrap>,
) -> Result<()> {
let init_message = HandshakeInitMessage {
addresses: HandshakeAddresses {
inputs: vec![input_address.to_string()],
outputs: vec![output_address.to_string()],
coordinator_input: coordinator.map(|c| c.input_address.clone()),
coordinator_output: coordinator.map(|c| c.output_address.clone()),
frontend_stats_publish_address: None,
},
parallel_config: Default::default(),
};
let payload = encode_msgpack(&init_message)?;
let message = ZmqMessage::try_from(vec![engine_id.to_frame(), Bytes::from(payload)])
.expect("handshake router messages must contain identity and payload");
handshake_socket.send(message).await?;
Ok(())
}
/// Receive the input registration message from each engine and validate its
/// identity.
///
/// Each registration contains 2 frames: `[identity, ready-payload]`.
///
/// Since vLLM commit `c8d98f81f676552c263f35bbde55e6edbe81b4e8` ("[Core]
/// Simplify API server handshake"), the payload is a msgpack-encoded
/// [`EngineCoreReadyResponse`] carrying post-initialization values such as
/// `max_model_len`.
///
/// Older engines sent an empty second frame here just to establish the
/// ROUTER/DEALER backchannel, with no structured payload on the input socket.
/// We continue to tolerate that legacy shape so the frontend can still connect
/// to slightly older local engine checkouts.
async fn wait_for_input_registrations(
input_socket: &mut RouterSocket,
engines: &mut [ConnectedEngine],
ready_timeout: Duration,
) -> Result<()> {
let mut pending = engines.iter().map(|e| e.engine_id.clone()).collect::<BTreeSet<_>>();
while !pending.is_empty() {
let registration = timeout(ready_timeout, input_socket.recv()).await.map_err(|_| {
Error::InputRegistrationTimeout {
timeout: ready_timeout,
}
})??;
if registration.len() != 2 {
bail_unexpected_handshake_message!(
"expected 2 frames for engine input registration, got {}",
registration.len()
);
}
let frames = registration.into_vec();
let actual_id = EngineId(frames[0].clone());
if !pending.remove(&actual_id) {
bail_unexpected_handshake_message!(
"received input registration for unexpected engine id {actual_id:?}"
);
}
let ready_response = if frames[1].is_empty() {
debug!(
?actual_id,
"received legacy empty input registration from engine"
);
None
} else {
let ready_response: EngineCoreReadyResponse = decode_msgpack(&frames[1])?;
debug!(
?actual_id,
?ready_response,
"received input registration from engine"
);
Some(ready_response)
};
// Store the ready response in the corresponding engine entry.
if let Some(engine) = engines.iter_mut().find(|e| e.engine_id == actual_id) {
engine.ready_response = ready_response;
}
}
Ok(())
}
/// Send an encoded message to the engine through the input socket.
pub async fn send_message(
input_send: &mut RouterSendHalf,
engine_id: &EngineId,
request_type: Bytes,
payload: Vec<u8>,
) -> Result<()> {
let message = ZmqMessage::try_from(vec![
engine_id.to_frame(),
request_type,
Bytes::from(payload),
])
.expect("router messages must contain identity and payload");
trace!(
?engine_id,
frame_count = message.len(),
"sending ZMQ message"
);
input_send.send(message).await?;
Ok(())
}
/// Run the output loop to receive messages from the engine and send them to the
/// provided channel.
pub async fn run_output_loop(
mut output_socket: PullSocket,
tx: mpsc::Sender<Result<EngineCoreOutputs>>,
) {
loop {
let message = match output_socket.recv().await {
Ok(message) => message,
Err(error) => {
// If we fail to receive a message from the engine, it's likely that the engine
// has crashed or become unreachable, so we should notify the
// client and shut down the output loop.
error!(error = %error.as_report(), "failed to receive output message");
let _ = tx.send(Err(Error::Transport(error))).await;
return;
}
};
let frame_count = message.len();
trace!(frame_count, "received output message");
let frames = message.into_vec();
let frame = frames.first().expect("output message must have at least one frame");
let frame_len = frame.len();
if frame.as_ref() == ENGINE_CORE_DEAD_SENTINEL {
warn!("received ENGINE_CORE_DEAD sentinel from engine");
let _ = tx.send(Err(Error::EngineCoreDead)).await;
return;
}
let decoded = match decode_engine_core_outputs(&frames) {
Ok(decoded) => {
trace!(frame_len, outputs = ?decoded, "decoded output message");
Ok(decoded)
}
Err(error) => {
// If we fail to decode the message from the engine, notify the client but keep
// the output loop running to continue processing future
// messages from the engine.
warn!(frame_len, error = %error.as_report(), "failed to decode output message");
Err(error)
}
};
if tx.send(decoded).await.is_err() {
// If we fail to send the decoded message to the client, it's likely that the
// client has shut down, so we should shut down the output loop as
// well.
warn!("output loop rx dropped, shutting down output loop");
return;
}
}
}
#[cfg(test)]
mod tests {
use super::bind_local_sockets;
#[tokio::test]
async fn bind_local_sockets_resolves_zero_port_bindings() {
let (input_address, _input_socket, output_address, _output_socket) =
bind_local_sockets("127.0.0.1", None, None).await.expect("bind local sockets");
assert!(input_address.starts_with("tcp://127.0.0.1:"));
assert!(output_address.starts_with("tcp://127.0.0.1:"));
assert_ne!(input_address, output_address);
}
}