[Rust Frontend] Move code from vllm-frontend-rs (#43283)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Signed-off-by: Eric Curtin <eric.curtin@docker.com> Signed-off-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com> Signed-off-by: Will.hou <1205157517@qq.com> Signed-off-by: Will.hou <willamhou@ceresman.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Eric Curtin <eric.curtin@docker.com> Co-authored-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com> Co-authored-by: Will.hou <1205157517@qq.com> Co-authored-by: Will.hou <willamhou@ceresman.com> Please see https://github.com/Inferact/vllm-frontend-rs for full original commit history.
This commit is contained in:
co-authored by
Nick Hill
Eric Curtin
Dev-X25874
Will.hou
Will.hou
parent
39d5fa96a7
commit
39910f2b25
@@ -0,0 +1,49 @@
|
||||
[package]
|
||||
name = "vllm-engine-core-client"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[features]
|
||||
test-util = ["dep:tempfile"]
|
||||
|
||||
[dependencies]
|
||||
arc-swap.workspace = true
|
||||
bytemuck.workspace = true
|
||||
byteorder.workspace = true
|
||||
bytes.workspace = true
|
||||
easy-ext.workspace = true
|
||||
enum-as-inner.workspace = true
|
||||
futures.workspace = true
|
||||
half.workspace = true
|
||||
hex.workspace = true
|
||||
itertools.workspace = true
|
||||
parking_lot.workspace = true
|
||||
rmp-serde.workspace = true
|
||||
rmpv.workspace = true
|
||||
serde.workspace = true
|
||||
serde_default.workspace = true
|
||||
serde_json.workspace = true
|
||||
serde_repr.workspace = true
|
||||
serde_tuple.workspace = true
|
||||
serde_with.workspace = true
|
||||
task-local.workspace = true
|
||||
tempfile = { workspace = true, optional = true }
|
||||
thiserror.workspace = true
|
||||
thiserror-ext.workspace = true
|
||||
tokio.workspace = true
|
||||
tokio-util.workspace = true
|
||||
tracing.workspace = true
|
||||
vllm-metrics.workspace = true
|
||||
zeromq.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
anyhow.workspace = true
|
||||
clap.workspace = true
|
||||
expect-test.workspace = true
|
||||
hex.workspace = true
|
||||
tempfile.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -0,0 +1,53 @@
|
||||
# Engine-Core Smoke Tests
|
||||
|
||||
Start headless `vllm`:
|
||||
|
||||
```bash
|
||||
source ../vllm/.venv/bin/activate
|
||||
HF_HUB_OFFLINE=1 \
|
||||
VLLM_LOGGING_LEVEL=DEBUG \
|
||||
VLLM_CPU_KVCACHE_SPACE=2 \
|
||||
VLLM_HOST_IP=127.0.0.1 \
|
||||
VLLM_LOOPBACK_IP=127.0.0.1 \
|
||||
python3 -m vllm.entrypoints.cli.main serve Qwen/Qwen3-0.6B \
|
||||
--headless \
|
||||
--enable-sleep-mode \
|
||||
--data-parallel-address 127.0.0.1 \
|
||||
--data-parallel-rpc-port 62100 \
|
||||
--data-parallel-size-local 1 \
|
||||
--max-model-len 512 \
|
||||
--dtype float16
|
||||
```
|
||||
|
||||
Run the Rust smoke test through the `vllm-engine-core-client` utility interface:
|
||||
|
||||
```bash
|
||||
cargo run -p vllm-engine-core-client --example external_engine_utility_call -- \
|
||||
--handshake-address tcp://127.0.0.1:62100 \
|
||||
--host 127.0.0.1
|
||||
```
|
||||
|
||||
If your current engine setup does not support sleep mode, skip the `sleep` / `wake_up` part of the
|
||||
smoke:
|
||||
|
||||
```bash
|
||||
cargo run -p vllm-engine-core-client --example external_engine_utility_call -- \
|
||||
--handshake-address tcp://127.0.0.1:62100 \
|
||||
--host 127.0.0.1 \
|
||||
--skip-sleep-wake
|
||||
```
|
||||
|
||||
Run the Rust smoke test for sample logprobs decoding through the raw engine-core request path:
|
||||
|
||||
```bash
|
||||
cargo run -p vllm-engine-core-client --example external_engine_logprobs -- \
|
||||
--handshake-address tcp://127.0.0.1:62100 \
|
||||
--host 127.0.0.1
|
||||
```
|
||||
|
||||
This smoke requests a small generated-token `logprobs` payload plus prompt logprobs over a much
|
||||
longer prompt, so it exercises both the inline and aux-frame decode paths against a real engine.
|
||||
The Rust client decodes those payloads into semantic per-position records rather than exposing the
|
||||
raw ndarray/tensor wire shape.
|
||||
|
||||
IMPORTANT: You must restart `vllm` each time you run the smoke test, as the vLLM engine cannot manage frontend closures and subsequent reconnects. In other words, do not reuse existing `vllm` instances, if any.
|
||||
@@ -0,0 +1,190 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use clap::Parser;
|
||||
use futures::StreamExt as _;
|
||||
use tokio::time::timeout;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use vllm_engine_core_client::protocol::{
|
||||
EngineCoreFinishReason, EngineCoreRequest, EngineCoreSamplingParams,
|
||||
};
|
||||
use vllm_engine_core_client::{
|
||||
EngineCoreClient, EngineCoreClientConfig, EngineCoreStreamOutput, TransportMode,
|
||||
};
|
||||
|
||||
const BASE_PROMPT_TOKEN_IDS: &[u32] = &[20841, 448, 6896, 25, 23811];
|
||||
|
||||
#[derive(Debug, Parser)]
|
||||
#[command(about = "Smoke-test engine-core sample logprobs against an external vLLM engine.")]
|
||||
struct Args {
|
||||
#[arg(long)]
|
||||
handshake_address: String,
|
||||
#[arg(long, default_value_t = 1)]
|
||||
engine_count: usize,
|
||||
#[arg(long, default_value = "Qwen/Qwen3-0.6B")]
|
||||
model: String,
|
||||
#[arg(long, default_value = "127.0.0.1")]
|
||||
host: String,
|
||||
#[arg(long, default_value_t = 0)]
|
||||
client_index: u32,
|
||||
#[arg(long, default_value_t = 30)]
|
||||
ready_timeout_secs: u64,
|
||||
#[arg(long, default_value_t = 120)]
|
||||
output_timeout_secs: u64,
|
||||
#[arg(long, default_value_t = 1)]
|
||||
max_tokens: u32,
|
||||
#[arg(long, default_value_t = 2)]
|
||||
logprobs: i32,
|
||||
#[arg(long, default_value_t = 1)]
|
||||
prompt_logprobs: i32,
|
||||
#[arg(long, default_value_t = 96)]
|
||||
prompt_repeats: usize,
|
||||
}
|
||||
|
||||
fn init_tracing() {
|
||||
let filter = EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| EnvFilter::new("vllm_engine_core_client=debug"));
|
||||
let _ = tracing_subscriber::fmt().with_env_filter(filter).try_init();
|
||||
}
|
||||
|
||||
fn unique_request_id() -> String {
|
||||
let nanos = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.expect("system clock should be after unix epoch")
|
||||
.as_nanos();
|
||||
format!("rust-engine-core-logprobs-{nanos}")
|
||||
}
|
||||
|
||||
fn build_prompt_token_ids(prompt_repeats: usize) -> Vec<u32> {
|
||||
let repeats = prompt_repeats.max(1);
|
||||
BASE_PROMPT_TOKEN_IDS.repeat(repeats)
|
||||
}
|
||||
|
||||
fn build_request(
|
||||
request_id: String,
|
||||
prompt_token_ids: Vec<u32>,
|
||||
max_tokens: u32,
|
||||
logprobs: i32,
|
||||
prompt_logprobs: i32,
|
||||
client_index: u32,
|
||||
) -> EngineCoreRequest {
|
||||
EngineCoreRequest {
|
||||
request_id,
|
||||
prompt_token_ids: Some(prompt_token_ids),
|
||||
sampling_params: Some(EngineCoreSamplingParams {
|
||||
max_tokens,
|
||||
logprobs: Some(logprobs),
|
||||
prompt_logprobs: Some(prompt_logprobs),
|
||||
..EngineCoreSamplingParams::for_test()
|
||||
}),
|
||||
arrival_time: 0.0,
|
||||
client_index,
|
||||
..EngineCoreRequest::default()
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_final_output(
|
||||
mut stream: vllm_engine_core_client::EngineCoreOutputStream,
|
||||
) -> Result<EngineCoreStreamOutput> {
|
||||
while let Some(output) = stream.next().await {
|
||||
let output = output.context("failed to receive engine-core output")?;
|
||||
if output.finished() {
|
||||
return Ok(output);
|
||||
}
|
||||
}
|
||||
bail!("request stream ended without a final output")
|
||||
}
|
||||
|
||||
#[tokio::main(flavor = "multi_thread")]
|
||||
async fn main() -> Result<()> {
|
||||
init_tracing();
|
||||
let args = Args::parse();
|
||||
let ready_timeout = Duration::from_secs(args.ready_timeout_secs);
|
||||
let output_timeout = Duration::from_secs(args.output_timeout_secs);
|
||||
let request_id = unique_request_id();
|
||||
let prompt_token_ids = build_prompt_token_ids(args.prompt_repeats);
|
||||
let client = EngineCoreClient::connect(EngineCoreClientConfig {
|
||||
transport_mode: TransportMode::HandshakeOwner {
|
||||
handshake_address: args.handshake_address.clone(),
|
||||
advertised_host: args.host.clone(),
|
||||
engine_count: args.engine_count,
|
||||
ready_timeout,
|
||||
local_input_address: None,
|
||||
local_output_address: None,
|
||||
},
|
||||
coordinator_mode: None,
|
||||
model_name: args.model.clone(),
|
||||
client_index: args.client_index,
|
||||
})
|
||||
.await
|
||||
.context("failed to connect to external vLLM engine")?;
|
||||
|
||||
println!("model={}", args.model);
|
||||
println!("handshake_address={}", args.handshake_address);
|
||||
println!("engine_count={}", args.engine_count);
|
||||
println!("input_address={}", client.input_address());
|
||||
println!("output_address={}", client.output_address());
|
||||
println!("engine_identities={:x?}", client.engine_identities());
|
||||
|
||||
let request = build_request(
|
||||
request_id.clone(),
|
||||
prompt_token_ids.clone(),
|
||||
args.max_tokens,
|
||||
args.logprobs,
|
||||
args.prompt_logprobs,
|
||||
args.client_index,
|
||||
);
|
||||
println!("request_id={request_id}");
|
||||
println!("prompt_len={}", prompt_token_ids.len());
|
||||
println!("base_prompt_len={}", BASE_PROMPT_TOKEN_IDS.len());
|
||||
println!("prompt_repeats={}", args.prompt_repeats);
|
||||
println!("requested_logprobs={}", args.logprobs);
|
||||
println!("requested_prompt_logprobs={}", args.prompt_logprobs);
|
||||
|
||||
let stream = client.call(request).await.context("failed to submit engine-core request")?;
|
||||
let output = timeout(output_timeout, wait_for_final_output(stream))
|
||||
.await
|
||||
.context("timed out waiting for final output")??;
|
||||
|
||||
let finish_reason = output.finish_reason;
|
||||
let token_ids = output.new_token_ids.clone();
|
||||
let logprobs = output
|
||||
.new_logprobs
|
||||
.as_ref()
|
||||
.and_then(|value| value.as_direct())
|
||||
.context("engine output did not include decoded sample logprobs")?;
|
||||
let prompt_logprobs = output
|
||||
.new_prompt_logprobs_tensors
|
||||
.as_ref()
|
||||
.and_then(|value| value.as_direct())
|
||||
.context("engine output did not include decoded prompt logprobs")?;
|
||||
|
||||
println!("token_ids={token_ids:?}");
|
||||
println!("finish_reason={finish_reason:?}");
|
||||
println!("new_logprobs={logprobs:#?}");
|
||||
println!("new_prompt_logprobs_tensors={prompt_logprobs:#?}");
|
||||
|
||||
client.shutdown().await.context("failed to shut down engine-core client")?;
|
||||
|
||||
if finish_reason != Some(EngineCoreFinishReason::Length) {
|
||||
bail!("unexpected finish_reason: expected Length, got {finish_reason:?}");
|
||||
}
|
||||
if token_ids.is_empty() {
|
||||
bail!("engine returned no generated token ids");
|
||||
}
|
||||
if logprobs.is_empty() {
|
||||
bail!("decoded logprobs payload is unexpectedly empty");
|
||||
}
|
||||
if prompt_logprobs.is_empty() {
|
||||
bail!("decoded prompt logprobs payload is unexpectedly empty");
|
||||
}
|
||||
if prompt_logprobs.len() + 1 < prompt_token_ids.len() {
|
||||
bail!(
|
||||
"prompt logprobs rows look too short: prompt_len={}, rows={}",
|
||||
prompt_token_ids.len(),
|
||||
prompt_logprobs.len()
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use clap::Parser;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, TransportMode};
|
||||
|
||||
#[derive(Debug, Parser)]
|
||||
#[command(about = "Smoke-test EngineCoreClient utility calls against an external vLLM engine.")]
|
||||
struct Args {
|
||||
#[arg(long)]
|
||||
handshake_address: String,
|
||||
#[arg(long, default_value_t = 1)]
|
||||
engine_count: usize,
|
||||
#[arg(long, default_value = "Qwen/Qwen3-0.6B")]
|
||||
model: String,
|
||||
#[arg(long, default_value = "127.0.0.1")]
|
||||
host: String,
|
||||
#[arg(long, default_value_t = 0)]
|
||||
client_index: u32,
|
||||
#[arg(long, default_value_t = 30)]
|
||||
ready_timeout_secs: u64,
|
||||
#[arg(
|
||||
long,
|
||||
default_value_t = false,
|
||||
help = "Expected initial result of is_sleeping() before running the smoke steps."
|
||||
)]
|
||||
expected_is_sleeping: bool,
|
||||
#[arg(long, default_value_t = false)]
|
||||
reset_running_requests: bool,
|
||||
#[arg(long, default_value_t = false)]
|
||||
reset_external: bool,
|
||||
#[arg(long, default_value_t = 1)]
|
||||
sleep_level: u32,
|
||||
#[arg(long, default_value = "abort")]
|
||||
sleep_mode: String,
|
||||
#[arg(
|
||||
long,
|
||||
default_value_t = false,
|
||||
help = "Skip sleep/wake_up calls when the engine was not started with sleep-mode support."
|
||||
)]
|
||||
skip_sleep_wake: bool,
|
||||
}
|
||||
|
||||
fn init_tracing() {
|
||||
let filter = EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| EnvFilter::new("vllm_engine_core_client=debug"));
|
||||
let _ = tracing_subscriber::fmt().with_env_filter(filter).try_init();
|
||||
}
|
||||
|
||||
#[tokio::main(flavor = "multi_thread")]
|
||||
async fn main() -> Result<()> {
|
||||
init_tracing();
|
||||
let args = Args::parse();
|
||||
let client = EngineCoreClient::connect(EngineCoreClientConfig {
|
||||
transport_mode: TransportMode::HandshakeOwner {
|
||||
handshake_address: args.handshake_address.clone(),
|
||||
advertised_host: args.host.clone(),
|
||||
engine_count: args.engine_count,
|
||||
ready_timeout: Duration::from_secs(args.ready_timeout_secs),
|
||||
local_input_address: None,
|
||||
local_output_address: None,
|
||||
},
|
||||
coordinator_mode: None,
|
||||
model_name: args.model.clone(),
|
||||
client_index: args.client_index,
|
||||
})
|
||||
.await
|
||||
.context("failed to connect to external vLLM engine")?;
|
||||
|
||||
println!("model={}", args.model);
|
||||
println!("handshake_address={}", args.handshake_address);
|
||||
println!("engine_count={}", args.engine_count);
|
||||
println!("input_address={}", client.input_address());
|
||||
println!("output_address={}", client.output_address());
|
||||
println!("engine_identities={:x?}", client.engine_identities());
|
||||
|
||||
let initial_is_sleeping =
|
||||
client.is_sleeping().await.context("failed to call is_sleeping utility")?;
|
||||
|
||||
println!("is_sleeping(initial)={initial_is_sleeping}");
|
||||
|
||||
if initial_is_sleeping != args.expected_is_sleeping {
|
||||
bail!(
|
||||
"unexpected initial is_sleeping state: expected {}, got {}",
|
||||
args.expected_is_sleeping,
|
||||
initial_is_sleeping
|
||||
);
|
||||
}
|
||||
|
||||
let reset_prefix_cache = client
|
||||
.reset_prefix_cache(args.reset_running_requests, args.reset_external)
|
||||
.await
|
||||
.context("failed to call reset_prefix_cache utility")?;
|
||||
println!("reset_prefix_cache={reset_prefix_cache}");
|
||||
|
||||
client.reset_mm_cache().await.context("failed to call reset_mm_cache utility")?;
|
||||
println!("reset_mm_cache=ok");
|
||||
|
||||
client
|
||||
.reset_encoder_cache()
|
||||
.await
|
||||
.context("failed to call reset_encoder_cache utility")?;
|
||||
println!("reset_encoder_cache=ok");
|
||||
|
||||
if args.skip_sleep_wake {
|
||||
println!("sleep_wake=skipped");
|
||||
} else {
|
||||
client.sleep(args.sleep_level, &args.sleep_mode).await.with_context(|| {
|
||||
format!(
|
||||
"failed to call sleep utility with level={} mode={}",
|
||||
args.sleep_level, args.sleep_mode
|
||||
)
|
||||
})?;
|
||||
println!(
|
||||
"sleep=ok level={} mode={}",
|
||||
args.sleep_level, args.sleep_mode
|
||||
);
|
||||
|
||||
let sleeping_after_sleep =
|
||||
client.is_sleeping().await.context("failed to call is_sleeping after sleep")?;
|
||||
println!("is_sleeping(after_sleep)={sleeping_after_sleep}");
|
||||
|
||||
if !sleeping_after_sleep {
|
||||
bail!("engine should report sleeping=true after sleep()");
|
||||
}
|
||||
|
||||
client.wake_up(None).await.context("failed to call wake_up utility")?;
|
||||
println!("wake_up=ok");
|
||||
|
||||
let sleeping_after_wake =
|
||||
client.is_sleeping().await.context("failed to call is_sleeping after wake_up")?;
|
||||
println!("is_sleeping(after_wake)={sleeping_after_wake}");
|
||||
|
||||
if sleeping_after_wake {
|
||||
bail!("engine should report sleeping=false after wake_up()");
|
||||
}
|
||||
}
|
||||
|
||||
client.shutdown().await.context("failed to shut down engine-core client")?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,673 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use futures::future::{join_all, try_join_all};
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_util::task::AbortOnDropHandle;
|
||||
use tracing::{debug, info, trace};
|
||||
|
||||
use crate::client::imp::{ClientInner, run_abort_loop, run_output_dispatcher_loop};
|
||||
use crate::coordinator::CoordinatorHandle;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::protocol::handshake::EngineCoreReadyResponse;
|
||||
use crate::protocol::{
|
||||
EngineCoreRequest, EngineCoreRequestType, EngineCoreUtilityRequest, ModelDtype,
|
||||
};
|
||||
use crate::transport::{self, ConnectedEngine};
|
||||
|
||||
pub(crate) mod imp;
|
||||
mod state;
|
||||
mod stream;
|
||||
|
||||
pub use stream::{EngineCoreOutputStream, EngineCoreStreamOutput};
|
||||
|
||||
/// How the frontend acquires its request/response transport with Python
|
||||
/// `EngineCoreProc`s.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum TransportMode {
|
||||
/// The Rust process owns the startup handshake and allocates or binds the
|
||||
/// frontend transport addresses itself before replying to engine
|
||||
/// `HELLO` messages.
|
||||
HandshakeOwner {
|
||||
/// Shared handshake endpoint that engines dial during startup.
|
||||
handshake_address: String,
|
||||
/// Host/IP that engines should use to connect back to the frontend
|
||||
/// transport sockets.
|
||||
advertised_host: String,
|
||||
/// Total number of engines expected to join this transport.
|
||||
engine_count: usize,
|
||||
/// Maximum time to wait for each startup phase to complete.
|
||||
ready_timeout: Duration,
|
||||
/// Optional explicit bind address for the input ROUTER socket.
|
||||
local_input_address: Option<String>,
|
||||
/// Optional explicit bind address for the output PULL socket.
|
||||
local_output_address: Option<String>,
|
||||
},
|
||||
|
||||
/// The Python supervisor has already chosen the frontend transport
|
||||
/// addresses, and the Rust process only needs to bind them and wait for
|
||||
/// engine registration frames.
|
||||
Bootstrapped {
|
||||
/// Input ROUTER socket address that engines will connect to for
|
||||
/// requests.
|
||||
input_address: String,
|
||||
/// Output PULL socket address that engines will connect to for
|
||||
/// responses.
|
||||
output_address: String,
|
||||
/// Total number of engines expected to register on this transport.
|
||||
engine_count: usize,
|
||||
/// Maximum time to wait for all expected engines to register.
|
||||
ready_timeout: Duration,
|
||||
},
|
||||
}
|
||||
|
||||
/// Which coordinator implementation should be active when one is present for a
|
||||
/// frontend client.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum CoordinatorMode {
|
||||
/// Run the Rust in-process coordinator for managed `serve` deployments.
|
||||
InProc,
|
||||
/// Connect to an external coordinator owned by another process.
|
||||
External { address: String },
|
||||
}
|
||||
|
||||
/// Configuration for connecting a Rust frontend client to an already running
|
||||
/// Python `EngineCoreProc`.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct EngineCoreClientConfig {
|
||||
/// Frontend-to-engine transport setup.
|
||||
pub transport_mode: TransportMode,
|
||||
/// Frontend-side coordinator behavior, or `None` when requests should flow
|
||||
/// directly to engines without any coordinator involvement.
|
||||
pub coordinator_mode: Option<CoordinatorMode>,
|
||||
/// Model name used for frontend-side metrics labels.
|
||||
pub model_name: String,
|
||||
/// Frontend client index stamped onto every request.
|
||||
pub client_index: u32,
|
||||
}
|
||||
|
||||
impl EngineCoreClientConfig {
|
||||
/// Create a new client config with the given handshake address, expecting a
|
||||
/// single engine, and default values for all other fields.
|
||||
pub fn new_single(handshake_address: impl Into<String>) -> Self {
|
||||
Self {
|
||||
transport_mode: TransportMode::HandshakeOwner {
|
||||
handshake_address: handshake_address.into(),
|
||||
advertised_host: "127.0.0.1".to_string(),
|
||||
engine_count: 1,
|
||||
ready_timeout: Duration::from_secs(30),
|
||||
local_input_address: None,
|
||||
local_output_address: None,
|
||||
},
|
||||
coordinator_mode: None,
|
||||
model_name: String::new(),
|
||||
client_index: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the model name used by frontend-side metrics and diagnostics.
|
||||
pub fn with_model_name(mut self, model_name: impl Into<String>) -> Self {
|
||||
self.model_name = model_name.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Override the client index stamped onto every outgoing request.
|
||||
pub fn with_client_index(mut self, client_index: u32) -> Self {
|
||||
self.client_index = client_index;
|
||||
self
|
||||
}
|
||||
|
||||
/// Override the optional coordinator mode for this client config.
|
||||
pub fn with_coordinator_mode(mut self, coordinator_mode: Option<CoordinatorMode>) -> Self {
|
||||
self.coordinator_mode = coordinator_mode;
|
||||
self
|
||||
}
|
||||
|
||||
/// Override the locally bound input/output addresses for handshake-owned
|
||||
/// transport mode.
|
||||
///
|
||||
/// This is primarily used by tests that want deterministic IPC endpoints
|
||||
/// while still exercising the handshake-owned startup path.
|
||||
pub fn with_local_input_output_addresses(
|
||||
mut self,
|
||||
local_input_address: Option<String>,
|
||||
local_output_address: Option<String>,
|
||||
) -> Self {
|
||||
let TransportMode::HandshakeOwner {
|
||||
local_input_address: current_input,
|
||||
local_output_address: current_output,
|
||||
..
|
||||
} = &mut self.transport_mode
|
||||
else {
|
||||
panic!("local input/output overrides are only valid in handshake-owned mode");
|
||||
};
|
||||
*current_input = local_input_address;
|
||||
*current_output = local_output_address;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// The reason a request stream is being aborted when its output stream is
|
||||
/// dropped.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum AbortCause {
|
||||
/// The consumer dropped the stream before the request reached a terminal
|
||||
/// engine output.
|
||||
#[default]
|
||||
DroppedStream,
|
||||
/// The frontend matched a stop string locally and intentionally stopped
|
||||
/// consuming the stream.
|
||||
StopStringMatched,
|
||||
}
|
||||
|
||||
task_local::task_local! {
|
||||
static ABORT_CAUSE: AbortCause;
|
||||
}
|
||||
|
||||
impl AbortCause {
|
||||
/// Return the abort cause currently associated with this task, or
|
||||
/// [`AbortCause::DroppedStream`] by default.
|
||||
pub fn current() -> Self {
|
||||
ABORT_CAUSE.try_get().unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Drop one value while marking the drop as happening for this abort cause.
|
||||
pub fn drop_as<T>(self, value: T) {
|
||||
ABORT_CAUSE.sync_scope(self, move || drop(value));
|
||||
}
|
||||
}
|
||||
|
||||
/// Internal auto-abort work item sent from stream `Drop` handlers to the abort
|
||||
/// worker.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct AbortRequest {
|
||||
request_id: String,
|
||||
cause: AbortCause,
|
||||
}
|
||||
|
||||
/// Default ZMQ-based implementation that talks directly to a Python
|
||||
/// `EngineCoreProc`.
|
||||
pub struct EngineCoreClient {
|
||||
config: EngineCoreClientConfig,
|
||||
input_address: String,
|
||||
output_address: String,
|
||||
engines: Vec<ConnectedEngine>,
|
||||
inner: Arc<ClientInner>,
|
||||
coordinator: Option<CoordinatorHandle>,
|
||||
abort_tx: mpsc::UnboundedSender<AbortRequest>,
|
||||
|
||||
// Background tasks
|
||||
output_task: AbortOnDropHandle<()>,
|
||||
dispatcher_task: AbortOnDropHandle<()>,
|
||||
abort_task: AbortOnDropHandle<()>,
|
||||
coordinator_output_task: Option<AbortOnDropHandle<()>>,
|
||||
coordinator_task: Option<AbortOnDropHandle<()>>,
|
||||
}
|
||||
|
||||
impl EngineCoreClient {
|
||||
/// Connect to Python `EngineCoreProc`s using the configured
|
||||
/// transport/coordinator modes.
|
||||
///
|
||||
/// In handshake-owned mode this method drives the full engine startup
|
||||
/// handshake. In bootstrapped mode it binds the provided frontend
|
||||
/// sockets and waits for the expected engine registration frames.
|
||||
pub async fn connect(config: EngineCoreClientConfig) -> Result<Self> {
|
||||
let connected = match &config.transport_mode {
|
||||
TransportMode::HandshakeOwner {
|
||||
handshake_address,
|
||||
advertised_host,
|
||||
engine_count,
|
||||
ready_timeout,
|
||||
local_input_address,
|
||||
local_output_address,
|
||||
} => {
|
||||
let enable_inproc_coordinator = match config.coordinator_mode {
|
||||
None => false,
|
||||
Some(CoordinatorMode::InProc) => true,
|
||||
Some(CoordinatorMode::External { .. }) => {
|
||||
return Err(Error::UnsupportedExternalCoordinator);
|
||||
}
|
||||
};
|
||||
|
||||
transport::connect_handshake(
|
||||
handshake_address,
|
||||
*engine_count,
|
||||
advertised_host,
|
||||
local_input_address.as_deref(),
|
||||
local_output_address.as_deref(),
|
||||
enable_inproc_coordinator,
|
||||
*ready_timeout,
|
||||
)
|
||||
.await?
|
||||
}
|
||||
|
||||
TransportMode::Bootstrapped {
|
||||
input_address,
|
||||
output_address,
|
||||
engine_count,
|
||||
ready_timeout,
|
||||
} => {
|
||||
if let Some(CoordinatorMode::InProc) = config.coordinator_mode {
|
||||
panic!("cannot use in-process coordinator with bootstrapped transport mode")
|
||||
}
|
||||
|
||||
transport::connect_bootstrapped(
|
||||
input_address,
|
||||
output_address,
|
||||
*engine_count,
|
||||
*ready_timeout,
|
||||
)
|
||||
.await?
|
||||
}
|
||||
};
|
||||
|
||||
Self::from_connected(config, connected).await
|
||||
}
|
||||
|
||||
/// Create a new client instance from the connected transport state after
|
||||
/// the startup handshake completes.
|
||||
async fn from_connected(
|
||||
config: EngineCoreClientConfig,
|
||||
connected: transport::ConnectedTransport,
|
||||
) -> Result<Self> {
|
||||
let (output_tx, output_rx) = mpsc::channel(64);
|
||||
let (abort_tx, abort_rx) = mpsc::unbounded_channel();
|
||||
let engines = connected.engines;
|
||||
let inner = Arc::new(ClientInner::new(
|
||||
connected.input_send,
|
||||
config.model_name.clone(),
|
||||
&engines,
|
||||
));
|
||||
let output_task = AbortOnDropHandle::new(tokio::spawn(transport::run_output_loop(
|
||||
connected.output_socket,
|
||||
output_tx,
|
||||
)));
|
||||
let dispatcher_task = AbortOnDropHandle::new(tokio::spawn(run_output_dispatcher_loop(
|
||||
inner.clone(),
|
||||
output_rx,
|
||||
)));
|
||||
let abort_task =
|
||||
AbortOnDropHandle::new(tokio::spawn(run_abort_loop(inner.clone(), abort_rx)));
|
||||
|
||||
// If any engine reported a dp_stats_address in its ready response, use it
|
||||
// as the external coordinator address.
|
||||
let dp_stats_address: Option<String> = engines
|
||||
.iter()
|
||||
.filter_map(|e| e.ready_response.as_ref())
|
||||
.find_map(|r| r.dp_stats_address.clone());
|
||||
|
||||
let (coordinator, coordinator_output_task, coordinator_task) =
|
||||
if let Some(coordinator_transport) = connected.coordinator {
|
||||
let (handle, runner) =
|
||||
CoordinatorHandle::new_inproc(coordinator_transport.input_socket);
|
||||
let (coordinator_output_tx, coordinator_output_rx) = mpsc::channel(64);
|
||||
let coordinator_output_task =
|
||||
AbortOnDropHandle::new(tokio::spawn(transport::run_output_loop(
|
||||
coordinator_transport.output_socket,
|
||||
coordinator_output_tx,
|
||||
)));
|
||||
let coordinator_task = AbortOnDropHandle::new(tokio::spawn(
|
||||
runner.run(coordinator_output_rx, inner.clone()),
|
||||
));
|
||||
(
|
||||
Some(handle),
|
||||
Some(coordinator_output_task),
|
||||
Some(coordinator_task),
|
||||
)
|
||||
} else if let Some(address) =
|
||||
dp_stats_address.as_deref().or(match config.coordinator_mode.as_ref() {
|
||||
Some(CoordinatorMode::External { address }) => Some(address.as_str()),
|
||||
_ => None,
|
||||
})
|
||||
{
|
||||
let (handle, service) = CoordinatorHandle::connect_external(address).await?;
|
||||
let coordinator_task =
|
||||
AbortOnDropHandle::new(tokio::spawn(service.run(inner.clone())));
|
||||
(Some(handle), None, Some(coordinator_task))
|
||||
} else {
|
||||
(None, None, None)
|
||||
};
|
||||
|
||||
Ok(Self {
|
||||
config,
|
||||
input_address: connected.input_address,
|
||||
output_address: connected.output_address,
|
||||
engines,
|
||||
inner,
|
||||
coordinator,
|
||||
abort_tx,
|
||||
output_task,
|
||||
dispatcher_task,
|
||||
abort_task,
|
||||
coordinator_output_task,
|
||||
coordinator_task,
|
||||
})
|
||||
}
|
||||
|
||||
/// Return the address of the input socket that the client uses to send
|
||||
/// requests to the engine.
|
||||
pub fn input_address(&self) -> &str {
|
||||
&self.input_address
|
||||
}
|
||||
|
||||
/// Return the address of the output socket that the client listens on for
|
||||
/// engine responses.
|
||||
pub fn output_address(&self) -> &str {
|
||||
&self.output_address
|
||||
}
|
||||
|
||||
/// Return the number of engines connected to this client.
|
||||
pub fn engine_count(&self) -> usize {
|
||||
self.engines.len()
|
||||
}
|
||||
|
||||
/// Return the engine identities of all engines connected to this client.
|
||||
pub fn engine_identities(&self) -> Vec<&[u8]> {
|
||||
self.engines.iter().map(|engine| &*engine.engine_id).collect()
|
||||
}
|
||||
|
||||
/// Return the ready responses received from all engines on the input
|
||||
/// socket.
|
||||
pub fn ready_responses(&self) -> Vec<&EngineCoreReadyResponse> {
|
||||
self.engines
|
||||
.iter()
|
||||
.filter_map(|engine| engine.ready_response.as_ref())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Return the engine-reported effective model dtype, when available.
|
||||
pub fn model_dtype(&self) -> Option<ModelDtype> {
|
||||
self.engines
|
||||
.iter()
|
||||
.filter_map(|engine| engine.ready_response.as_ref())
|
||||
.find_map(|response| response.dtype)
|
||||
}
|
||||
|
||||
/// Return the total number of GPU blocks summed across all connected
|
||||
/// engines.
|
||||
pub fn total_num_gpu_blocks(&self) -> u64 {
|
||||
self.engines
|
||||
.iter()
|
||||
.filter_map(|engine| engine.ready_response.as_ref())
|
||||
.map(|r| r.num_gpu_blocks)
|
||||
.sum()
|
||||
}
|
||||
|
||||
/// Return the minimum engine-reported `max_model_len` across all engines.
|
||||
///
|
||||
/// This is the auto-fitted value after KV cache profiling and may differ
|
||||
/// from the originally configured value.
|
||||
pub fn max_model_len(&self) -> Option<u32> {
|
||||
self.engines
|
||||
.iter()
|
||||
.filter_map(|e| e.ready_response.as_ref())
|
||||
.map(|r| r.max_model_len as u32)
|
||||
.min()
|
||||
}
|
||||
|
||||
/// Get the model name associated with this client used for metrics
|
||||
/// labeling.
|
||||
pub fn model_name(&self) -> &str {
|
||||
self.inner.model_name()
|
||||
}
|
||||
|
||||
/// Return whether the client still considers the engine healthy.
|
||||
pub fn is_healthy(&self) -> bool {
|
||||
self.inner.is_healthy()
|
||||
}
|
||||
|
||||
/// Return the first persistent health error observed by the client, if any.
|
||||
pub fn health_error(&self) -> Option<Arc<Error>> {
|
||||
self.inner.health_error()
|
||||
}
|
||||
}
|
||||
|
||||
// Client API implementation.
|
||||
impl EngineCoreClient {
|
||||
/// Add a new request to the engine and return a per-request raw output
|
||||
/// stream.
|
||||
pub async fn call(&self, mut req: EngineCoreRequest) -> Result<EngineCoreOutputStream> {
|
||||
req.client_index = self.config.client_index;
|
||||
req.validate()?;
|
||||
trace!(
|
||||
request_id = %req.request_id,
|
||||
client_index = req.client_index,
|
||||
current_wave = req.current_wave,
|
||||
request = ?req,
|
||||
"sending add request"
|
||||
);
|
||||
|
||||
let request_id = req.request_id.clone();
|
||||
let data_parallel_rank = req.data_parallel_rank;
|
||||
let (engine_id, rx) =
|
||||
self.inner.register_request(request_id.clone(), data_parallel_rank)?;
|
||||
|
||||
let result: Result<()> = async {
|
||||
if let Some(coordinator) = self.coordinator.as_ref() {
|
||||
let snapshot = coordinator.snapshot();
|
||||
req.current_wave = snapshot.current_wave;
|
||||
if !snapshot.engines_running {
|
||||
coordinator.notify_first_request(engine_id.clone())?;
|
||||
}
|
||||
}
|
||||
|
||||
debug!(
|
||||
request_id = req.request_id,
|
||||
?engine_id,
|
||||
"registered request to engine"
|
||||
);
|
||||
|
||||
self.inner.send_to_engine(&engine_id, EngineCoreRequestType::Add, &req).await?;
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
|
||||
// Failed to send the request to the engine, roll back the registration.
|
||||
if let Err(error) = result {
|
||||
self.inner.rollback_request(&request_id);
|
||||
return Err(error);
|
||||
}
|
||||
|
||||
Ok(EngineCoreOutputStream::new(
|
||||
request_id,
|
||||
self.abort_tx.clone(),
|
||||
rx,
|
||||
))
|
||||
}
|
||||
|
||||
/// Abort currently in-flight requests by request ID.
|
||||
pub async fn abort(&self, ids: &[String]) -> Result<()> {
|
||||
let abortable = self.inner.abortable_request_ids(ids)?;
|
||||
|
||||
trace!(request_ids = ?ids, abortable_request_ids = ?abortable, "sending abort request ids");
|
||||
|
||||
if abortable.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
for (engine_id, request_ids) in abortable {
|
||||
self.inner.do_abort_requests(&engine_id, &request_ids).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Call a typed utility method on all connected engines, returning one
|
||||
/// decoded result per connected engine if all calls succeed or an error
|
||||
/// if any call fails.
|
||||
///
|
||||
/// Callers should pass utility arguments using Rust tuple semantics so the
|
||||
/// encoded payload matches Python's `(client_index, call_id,
|
||||
/// method_name, args)` contract: `()`, `(arg,)`, `(arg1, arg2)`, etc.
|
||||
pub async fn call_utility<T, A>(&self, method: &str, args: A) -> Result<Vec<T>>
|
||||
where
|
||||
T: serde::de::DeserializeOwned,
|
||||
A: serde::Serialize + std::fmt::Debug,
|
||||
{
|
||||
trace!(
|
||||
method,
|
||||
client_index = self.config.client_index,
|
||||
engine_count = self.engines.len(),
|
||||
"sending utility request"
|
||||
);
|
||||
|
||||
// Phase 1: allocate one call id per engine and build the per-engine
|
||||
// request payloads up-front. Any failure here (registry closed, encode
|
||||
// error) must roll back the call ids already allocated so they do not
|
||||
// leak in the utility registry until shutdown.
|
||||
let mut pending_calls = Vec::with_capacity(self.engines.len());
|
||||
let mut prepared_sends = Vec::with_capacity(self.engines.len());
|
||||
for engine in &self.engines {
|
||||
let (call_id, rx) = match self.inner.allocate_and_register_utility_call() {
|
||||
Ok(pair) => pair,
|
||||
Err(err) => {
|
||||
self.inner.unregister_utility_calls(pending_calls.iter().map(|(id, _)| *id));
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
let request = match EngineCoreUtilityRequest::new(
|
||||
self.config.client_index,
|
||||
call_id,
|
||||
method,
|
||||
&args,
|
||||
) {
|
||||
Ok(request) => request,
|
||||
Err(err) => {
|
||||
self.inner.unregister_utility_calls(
|
||||
pending_calls.iter().map(|(id, _)| *id).chain(std::iter::once(call_id)),
|
||||
);
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
pending_calls.push((call_id, rx));
|
||||
prepared_sends.push((&engine.engine_id, request));
|
||||
}
|
||||
|
||||
// Phase 2: dispatch every utility request concurrently. `try_join_all`
|
||||
// fails fast on the first transport error and drops the remaining send
|
||||
// futures; any engines that already received the request will reply,
|
||||
// but those replies are simply dropped because we roll back the call
|
||||
// ids below.
|
||||
let send_futures = prepared_sends.iter().map(|(engine_id, request)| {
|
||||
self.inner.send_to_engine(engine_id, EngineCoreRequestType::Utility, request)
|
||||
});
|
||||
if let Err(err) = try_join_all(send_futures).await {
|
||||
self.inner.unregister_utility_calls(pending_calls.iter().map(|(id, _)| *id));
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
// Phase 3: wait for all engines to respond and preserve the per-engine
|
||||
// result list.
|
||||
let futures = pending_calls.into_iter().map(|(call_id, rx)| async move {
|
||||
rx.await
|
||||
.map_err(|_| Error::UtilityCallClosed {
|
||||
method: method.to_string(),
|
||||
call_id,
|
||||
})??
|
||||
.into_typed_result(method)
|
||||
});
|
||||
try_join_all(futures).await
|
||||
}
|
||||
|
||||
/// Execute `collective_rpc` on all engines and flatten all engine results
|
||||
/// into one list.
|
||||
pub async fn collective_rpc<A, K>(
|
||||
&self,
|
||||
method: &str,
|
||||
timeout: Option<f64>,
|
||||
args: A,
|
||||
kwargs: K,
|
||||
) -> Result<Vec<rmpv::Value>>
|
||||
where
|
||||
A: serde::Serialize + std::fmt::Debug,
|
||||
K: serde::Serialize + std::fmt::Debug,
|
||||
{
|
||||
let results = self
|
||||
.call_utility::<rmpv::Value, _>("collective_rpc", (method, timeout, args, kwargs))
|
||||
.await?;
|
||||
|
||||
Ok(results
|
||||
.into_iter()
|
||||
.flat_map(|result| match result {
|
||||
// Each engine's `collective_rpc` result is itself the worker-level result list.
|
||||
rmpv::Value::Array(results) => results,
|
||||
other => vec![other],
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Return whether the engine is currently sleeping at any level.
|
||||
pub async fn is_sleeping(&self) -> Result<bool> {
|
||||
// TODO: we only return the result of the first engine here.
|
||||
Ok(self.call_utility("is_sleeping", ()).await?[0])
|
||||
}
|
||||
|
||||
/// Reset the multi-modal cache.
|
||||
pub async fn reset_mm_cache(&self) -> Result<()> {
|
||||
self.call_utility::<(), _>("reset_mm_cache", ()).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reset the encoder cache.
|
||||
pub async fn reset_encoder_cache(&self) -> Result<()> {
|
||||
self.call_utility::<(), _>("reset_encoder_cache", ()).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reset the prefix cache and optionally the external connector cache.
|
||||
pub async fn reset_prefix_cache(
|
||||
&self,
|
||||
reset_running_requests: bool,
|
||||
reset_connector: bool,
|
||||
) -> Result<bool> {
|
||||
// TODO: we only return the result of the first engine here.
|
||||
Ok(self
|
||||
.call_utility(
|
||||
"reset_prefix_cache",
|
||||
(reset_running_requests, reset_connector),
|
||||
)
|
||||
.await?[0])
|
||||
}
|
||||
|
||||
/// Put the engine to sleep.
|
||||
pub async fn sleep(&self, level: u32, mode: &str) -> Result<()> {
|
||||
self.call_utility::<(), _>("sleep", (level, mode)).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Wake the engine from sleep, optionally limiting the wake-up to specific
|
||||
/// tags.
|
||||
pub async fn wake_up(&self, tags: Option<Vec<String>>) -> Result<()> {
|
||||
self.call_utility::<(), _>("wake_up", (tags,)).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Shut down local client tasks and close transport state.
|
||||
pub async fn shutdown(self) -> Result<()> {
|
||||
let Self {
|
||||
inner,
|
||||
abort_tx,
|
||||
output_task,
|
||||
dispatcher_task,
|
||||
abort_task,
|
||||
coordinator_output_task,
|
||||
coordinator_task,
|
||||
..
|
||||
} = self;
|
||||
|
||||
info!("shutting down engine-core client");
|
||||
inner.shutdown();
|
||||
drop(abort_tx);
|
||||
|
||||
// Abort all client tasks first, then await them.
|
||||
// Note the aborting orders here.
|
||||
let mut tasks = vec![abort_task, dispatcher_task, output_task];
|
||||
tasks.extend(coordinator_task);
|
||||
tasks.extend(coordinator_output_task);
|
||||
|
||||
tasks.iter().for_each(|t| t.abort());
|
||||
join_all(tasks).await;
|
||||
|
||||
info!("engine-core client shut down");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,410 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::slice;
|
||||
use std::sync::Arc;
|
||||
|
||||
use arc_swap::ArcSwapOption;
|
||||
use parking_lot::Mutex;
|
||||
use thiserror_ext::AsReport as _;
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::{debug, info, trace, warn};
|
||||
use vllm_metrics::METRICS;
|
||||
use zeromq::RouterSendHalf;
|
||||
|
||||
use crate::client::state::{OutputReceiver, RequestRegistry, UtilityReceiver, UtilityRegistry};
|
||||
use crate::client::stream::EngineCoreStreamOutput;
|
||||
use crate::client::{AbortCause, AbortRequest};
|
||||
use crate::error::{client_closed, dispatcher_closed, unexpected_dispatcher_output};
|
||||
use crate::metrics::record_scheduler_stats;
|
||||
use crate::protocol::stats::SchedulerStats;
|
||||
use crate::protocol::{
|
||||
ClassifiedEngineCoreOutputs, EngineCoreOutput, EngineCoreOutputs, EngineCoreRequestType,
|
||||
UtilityOutput, encode_msgpack,
|
||||
};
|
||||
use crate::transport::{ConnectedEngine, EngineId};
|
||||
use crate::{Error, Result, transport};
|
||||
|
||||
pub(crate) struct ClientInner {
|
||||
input_send: RouterSendHalf,
|
||||
model_name: String,
|
||||
request_reg: Mutex<RequestRegistry>,
|
||||
utility_reg: Mutex<UtilityRegistry>,
|
||||
health_error: ArcSwapOption<Error>,
|
||||
}
|
||||
|
||||
impl ClientInner {
|
||||
/// Create a new instance with the given input send half after the startup
|
||||
/// handshake completes.
|
||||
pub fn new(
|
||||
input_send: RouterSendHalf,
|
||||
model_name: String,
|
||||
engines: &[ConnectedEngine],
|
||||
) -> Self {
|
||||
Self {
|
||||
input_send,
|
||||
model_name,
|
||||
request_reg: Mutex::new(RequestRegistry::new(engines)),
|
||||
utility_reg: Mutex::new(UtilityRegistry::default()),
|
||||
health_error: ArcSwapOption::empty(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the model name associated with this client used for metrics
|
||||
/// labeling.
|
||||
pub fn model_name(&self) -> &str {
|
||||
&self.model_name
|
||||
}
|
||||
|
||||
/// Register a newly added request. Return the selected engine id and the
|
||||
/// per-request output channel bound to its `request_id`.
|
||||
///
|
||||
/// When `data_parallel_rank` is provided, the request is routed to that
|
||||
/// specific engine rank, bypassing load balancing.
|
||||
pub fn register_request(
|
||||
&self,
|
||||
request_id: String,
|
||||
data_parallel_rank: Option<u32>,
|
||||
) -> Result<(EngineId, OutputReceiver)> {
|
||||
let mut registry = self.request_reg.lock();
|
||||
if registry.is_closed() {
|
||||
return Err(self.closed_error());
|
||||
}
|
||||
registry.register(request_id, data_parallel_rank)
|
||||
}
|
||||
|
||||
/// Allocate the next utility `call_id` and register its waiting receiver.
|
||||
pub fn allocate_and_register_utility_call(&self) -> Result<(i64, UtilityReceiver)> {
|
||||
let mut registry = self.utility_reg.lock();
|
||||
if registry.is_closed() {
|
||||
return Err(self.closed_error());
|
||||
}
|
||||
Ok(registry.allocate_and_register())
|
||||
}
|
||||
|
||||
/// Undo a batch of utility call allocations when the fan-out send fails
|
||||
/// partway through. Silently ignores unknown call ids so callers can pass
|
||||
/// the full set without first filtering successful sends.
|
||||
pub fn unregister_utility_calls(&self, call_ids: impl IntoIterator<Item = i64>) {
|
||||
self.utility_reg.lock().unregister_many(call_ids);
|
||||
}
|
||||
|
||||
/// Undo a request registration when `add_request()` fails.
|
||||
pub fn rollback_request(&self, request_id: &str) {
|
||||
let _ = self.request_reg.lock().remove(request_id);
|
||||
}
|
||||
|
||||
/// Filter the given request IDs to the subset that are still tracked as
|
||||
/// active and can be aborted, grouped by the engine that originally
|
||||
/// accepted them.
|
||||
pub fn abortable_request_ids(
|
||||
&self,
|
||||
request_ids: &[String],
|
||||
) -> Result<BTreeMap<EngineId, Vec<String>>> {
|
||||
let registry = self.request_reg.lock();
|
||||
if registry.is_closed() {
|
||||
return Err(self.closed_error());
|
||||
}
|
||||
Ok(registry.abortable_request_ids(request_ids))
|
||||
}
|
||||
|
||||
/// Obtain the stream sender for one output. If it indicates the request is
|
||||
/// finished, it will be removed from the registry.
|
||||
pub fn take_sender_for_output(
|
||||
&self,
|
||||
output: &EngineCoreOutput,
|
||||
) -> Option<mpsc::UnboundedSender<Result<EngineCoreStreamOutput>>> {
|
||||
self.request_reg.lock().sender_for_output(output)
|
||||
}
|
||||
|
||||
/// Remove a batch of requests that have finished or aborted, returning
|
||||
/// their stream senders.
|
||||
pub fn finish_requests<'a>(
|
||||
&self,
|
||||
request_ids: impl IntoIterator<Item = &'a String>,
|
||||
) -> Vec<mpsc::UnboundedSender<Result<EngineCoreStreamOutput>>> {
|
||||
self.request_reg.lock().finish_many(request_ids)
|
||||
}
|
||||
|
||||
/// Apply one scheduler stats update for the given engine to the local
|
||||
/// routing state. Returns `false` if the engine is unknown to the
|
||||
/// client.
|
||||
pub fn apply_scheduler_stats(&self, engine_index: u32, stats: &SchedulerStats) -> bool {
|
||||
self.request_reg.lock().apply_scheduler_stats(engine_index, stats)
|
||||
}
|
||||
|
||||
/// Close all active request streams and utility calls with the first
|
||||
/// persistent health error.
|
||||
pub fn close_registries(&self, error: Arc<Error>) {
|
||||
let persistent_error = self.record_health_error(error);
|
||||
let request_senders = self.request_reg.lock().close();
|
||||
let utility_senders = self.utility_reg.lock().close();
|
||||
|
||||
// Notify all ongoing requests that the client is closed.
|
||||
for sender in request_senders {
|
||||
let _ = sender.send(Err(Error::Shared(persistent_error.clone())));
|
||||
}
|
||||
for sender in utility_senders {
|
||||
let _ = sender.send(Err(Error::Shared(persistent_error.clone())));
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the first persistent health error observed by the client, if any.
|
||||
pub fn health_error(&self) -> Option<Arc<Error>> {
|
||||
self.health_error.load_full()
|
||||
}
|
||||
|
||||
/// Return whether the client still considers the engine healthy.
|
||||
pub fn is_healthy(&self) -> bool {
|
||||
self.health_error.load().is_none()
|
||||
}
|
||||
|
||||
/// Resolve one utility output to the waiting caller. Returns `true` if a
|
||||
/// waiting caller existed.
|
||||
pub fn resolve_utility_output(&self, output: UtilityOutput) -> bool {
|
||||
match self.utility_reg.lock().resolve(&output.call_id) {
|
||||
Some(sender) => {
|
||||
sender.send(Ok(output)).unwrap_or_default();
|
||||
true
|
||||
}
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Send the given message to the engine. The request should be first
|
||||
/// registered via `register_request()` to ensure the request stream is
|
||||
/// tracked.
|
||||
pub async fn send_to_engine<T>(
|
||||
&self,
|
||||
engine_id: &EngineId,
|
||||
request_type: EngineCoreRequestType,
|
||||
payload: &T,
|
||||
) -> Result<()>
|
||||
where
|
||||
T: serde::Serialize + std::fmt::Debug,
|
||||
{
|
||||
// TODO: for `EngineCoreRequest`, split outbound tensor raw views into aux
|
||||
// frames instead of always producing a single msgpack frame.
|
||||
let payload = encode_msgpack(payload)?;
|
||||
let mut input_send = self.input_send.clone();
|
||||
transport::send_message(&mut input_send, engine_id, request_type.to_frame(), payload)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Handle an abort request by sending the abort message to the engine.
|
||||
pub async fn do_abort_requests(
|
||||
&self,
|
||||
engine_id: &EngineId,
|
||||
request_ids: &[String],
|
||||
) -> Result<()> {
|
||||
self.send_to_engine(engine_id, EngineCoreRequestType::Abort, &request_ids).await
|
||||
}
|
||||
|
||||
/// Shut down by closing all active request streams and utility calls with a
|
||||
/// sticky client closed error.
|
||||
pub fn shutdown(&self) {
|
||||
self.close_registries(Arc::new(client_closed!("engine-core client shut down")));
|
||||
}
|
||||
|
||||
/// Remove the request from the active registry for auto-abort and return
|
||||
/// the engine that the request was originally routed to, if it is still
|
||||
/// active.
|
||||
pub fn take_auto_abort_target(&self, request_id: &str) -> Option<EngineId> {
|
||||
let mut registry = self.request_reg.lock();
|
||||
let (_, engine_id) = registry.remove(request_id)?;
|
||||
if registry.is_closed() {
|
||||
return None;
|
||||
}
|
||||
Some(engine_id)
|
||||
}
|
||||
|
||||
/// Publish the first persistent health error and return the sticky error
|
||||
/// recorded for this client. Later failures do not overwrite the first
|
||||
/// one so `/health` and post-close callers observe a stable cause.
|
||||
fn record_health_error(&self, error: Arc<Error>) -> Arc<Error> {
|
||||
if let Some(existing) = self.health_error.load_full() {
|
||||
return existing;
|
||||
}
|
||||
self.health_error
|
||||
.rcu(|current| current.clone().unwrap_or_else(|| error.clone()));
|
||||
self.health_error
|
||||
.load_full()
|
||||
.expect("health error must be recorded before registries close")
|
||||
}
|
||||
|
||||
/// Assert there is a recorded health error and return a `Shared` variant
|
||||
/// wrapping it for error returns when the client is already closed.
|
||||
fn closed_error(&self) -> Error {
|
||||
Error::Shared(self.health_error.load_full().expect(
|
||||
"closed registry must have a recorded health error before rejecting new operations",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// Background loop that listens for request IDs to abort and sends abort
|
||||
/// messages to the engine. This is used to implement the auto-abort behavior
|
||||
/// when a request stream is dropped without being properly terminated.
|
||||
pub(crate) async fn run_abort_loop(
|
||||
inner: Arc<ClientInner>,
|
||||
mut abort_rx: mpsc::UnboundedReceiver<AbortRequest>,
|
||||
) {
|
||||
// TODO: receive and abort requests in batch
|
||||
while let Some(AbortRequest { request_id, cause }) = abort_rx.recv().await {
|
||||
let Some(engine_id) = inner.take_auto_abort_target(&request_id) else {
|
||||
debug!(request_id, "skip auto-abort for inactive request");
|
||||
continue;
|
||||
};
|
||||
|
||||
match cause {
|
||||
AbortCause::DroppedStream => {
|
||||
info!(request_id, "auto-aborting request due to dropped stream")
|
||||
}
|
||||
AbortCause::StopStringMatched => {
|
||||
debug!(
|
||||
request_id,
|
||||
"auto-aborting request due to stop string matched"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if let Err(error) = inner.do_abort_requests(&engine_id, slice::from_ref(&request_id)).await
|
||||
{
|
||||
warn!(
|
||||
request_id,
|
||||
?engine_id,
|
||||
error = %error.as_report(),
|
||||
"failed to auto-abort dropped request stream"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Background loop that listens for engine-core outputs and dispatches them to
|
||||
/// the corresponding request streams based on their `request_id`.
|
||||
pub(crate) async fn run_output_dispatcher_loop(
|
||||
inner: Arc<ClientInner>,
|
||||
mut output_rx: mpsc::Receiver<Result<EngineCoreOutputs>>,
|
||||
) {
|
||||
let result: Result<()> = async {
|
||||
loop {
|
||||
let outputs = match output_rx.recv().await {
|
||||
Some(outputs) => outputs,
|
||||
None => Err(dispatcher_closed!(
|
||||
"engine-core output dispatcher channel closed"
|
||||
)),
|
||||
}?;
|
||||
|
||||
match outputs.classify() {
|
||||
ClassifiedEngineCoreOutputs::RequestBatch(batch) => {
|
||||
for output in batch.outputs {
|
||||
let request_id = output.request_id.clone();
|
||||
let Some(sender) = inner.take_sender_for_output(&output) else {
|
||||
debug!(request_id, "dropping output for inactive request");
|
||||
continue;
|
||||
};
|
||||
|
||||
let wrapped_output = EngineCoreStreamOutput {
|
||||
engine_index: batch.engine_index,
|
||||
timestamp: batch.timestamp,
|
||||
output,
|
||||
};
|
||||
if sender.send(Ok(wrapped_output)).is_err() {
|
||||
debug!(request_id, "request output stream receiver dropped");
|
||||
}
|
||||
}
|
||||
|
||||
// The sender for normally-finished requests should have already been removed
|
||||
// from the registry when their final output was dispatched
|
||||
// above. This serves as a safety net to capture any
|
||||
// requests marked as finished by the engine.
|
||||
if let Some(finished_requests) = batch.finished_requests.as_ref() {
|
||||
for request_id in finished_requests {
|
||||
trace!(request_id, "request completed via finished_requests");
|
||||
}
|
||||
drop(inner.finish_requests(finished_requests));
|
||||
}
|
||||
|
||||
if let Some(scheduler_stats) = batch.scheduler_stats.as_ref() {
|
||||
if !inner.apply_scheduler_stats(batch.engine_index, scheduler_stats) {
|
||||
debug!(
|
||||
engine_index = batch.engine_index,
|
||||
"dropping scheduler stats for unknown engine"
|
||||
);
|
||||
}
|
||||
record_scheduler_stats(
|
||||
&METRICS.scheduler,
|
||||
inner.model_name(),
|
||||
batch.engine_index,
|
||||
scheduler_stats,
|
||||
);
|
||||
}
|
||||
}
|
||||
ClassifiedEngineCoreOutputs::Utility(utility) => {
|
||||
let call_id = utility.output.call_id;
|
||||
if inner.resolve_utility_output(utility.output) {
|
||||
trace!(
|
||||
call_id,
|
||||
engine_index = utility.engine_index,
|
||||
"resolved utility output"
|
||||
);
|
||||
} else {
|
||||
warn!(
|
||||
call_id,
|
||||
engine_index = utility.engine_index,
|
||||
"dropping output for unexpected utility call"
|
||||
);
|
||||
}
|
||||
}
|
||||
other @ (ClassifiedEngineCoreOutputs::DpControl { .. }
|
||||
| ClassifiedEngineCoreOutputs::Other(_)) => {
|
||||
Err::<(), _>(unexpected_dispatcher_output!(
|
||||
"received unexpected output on main dispatcher path: {other:?}"
|
||||
))?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
.await;
|
||||
let Err(error) = result else { return };
|
||||
|
||||
warn!(error = %error.as_report(), "output dispatcher exiting with error");
|
||||
inner.close_registries(Arc::new(error));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use zeromq::{RouterSocket, Socket};
|
||||
|
||||
use super::*;
|
||||
|
||||
async fn test_inner() -> ClientInner {
|
||||
let mut socket = RouterSocket::new();
|
||||
socket.bind("tcp://127.0.0.1:0").await.unwrap();
|
||||
let (send, _) = socket.split();
|
||||
ClientInner::new(
|
||||
send,
|
||||
"test-model".to_string(),
|
||||
&[ConnectedEngine {
|
||||
engine_id: EngineId::from(b"engine-0"),
|
||||
ready_response: None,
|
||||
}],
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn close_registries_records_first_health_error_only() {
|
||||
let inner = test_inner().await;
|
||||
|
||||
inner.close_registries(Arc::new(Error::EngineCoreDead));
|
||||
assert!(!inner.is_healthy());
|
||||
assert!(matches!(
|
||||
inner.health_error().as_deref(),
|
||||
Some(Error::EngineCoreDead)
|
||||
));
|
||||
|
||||
inner.close_registries(Arc::new(client_closed!("shutdown")));
|
||||
assert!(matches!(
|
||||
inner.health_error().as_deref(),
|
||||
Some(Error::EngineCoreDead)
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,670 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::atomic::{AtomicI64, Ordering};
|
||||
|
||||
use tokio::sync::{mpsc, oneshot};
|
||||
use tracing::trace;
|
||||
|
||||
use crate::EngineId;
|
||||
use crate::client::stream::EngineCoreStreamOutput;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::protocol::stats::SchedulerStats;
|
||||
use crate::protocol::{EngineCoreOutput, UtilityOutput};
|
||||
use crate::transport::ConnectedEngine;
|
||||
|
||||
pub type OutputSender = mpsc::UnboundedSender<Result<EngineCoreStreamOutput>>;
|
||||
pub type OutputReceiver = mpsc::UnboundedReceiver<Result<EngineCoreStreamOutput>>;
|
||||
pub type UtilitySender = oneshot::Sender<Result<UtilityOutput>>;
|
||||
pub type UtilityReceiver = oneshot::Receiver<Result<UtilityOutput>>;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct TrackedRequest {
|
||||
sender: OutputSender,
|
||||
engine_id: EngineId,
|
||||
}
|
||||
|
||||
/// The latest real scheduler-side load snapshot observed from one engine.
|
||||
///
|
||||
/// These counters come from `scheduler_stats` on the normal engine output path
|
||||
/// and are the preferred routing signal once available.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
struct EngineLoadSnapshot {
|
||||
/// Requests still counted on the scheduler's waiting side.
|
||||
waiting: usize,
|
||||
/// Requests currently counted on the scheduler's running side.
|
||||
running: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct EngineRoutingState {
|
||||
/// Requests admitted by this frontend that have not finished yet.
|
||||
///
|
||||
/// This is used both as the bootstrap fallback before real scheduler stats
|
||||
/// exist and as a lower bound afterwards so asynchronous scheduler
|
||||
/// snapshots cannot erase frontend admission history.
|
||||
inflight: usize,
|
||||
/// The latest real scheduler snapshot received from this engine, if any.
|
||||
last_scheduler_stats: Option<EngineLoadSnapshot>,
|
||||
}
|
||||
|
||||
impl EngineRoutingState {
|
||||
/// Compute the routing score used to pick the least-loaded engine.
|
||||
///
|
||||
/// Scheduler stats can raise the load estimate above the frontend-local
|
||||
/// view, but they should not lower it below requests this frontend has
|
||||
/// already admitted. Waiting requests still get the same extra penalty
|
||||
/// as the original `waiting * 4 + running` score.
|
||||
fn routing_score(&self) -> usize {
|
||||
const WAITING_WEIGHT: usize = 4;
|
||||
|
||||
let Some(stats) = self.last_scheduler_stats else {
|
||||
return self.inflight;
|
||||
};
|
||||
|
||||
let scheduler_total = stats.running + stats.waiting;
|
||||
self.inflight.max(scheduler_total) + stats.waiting * (WAITING_WEIGHT - 1)
|
||||
}
|
||||
|
||||
/// Replace the local routing view with a fresh real scheduler snapshot.
|
||||
fn apply_scheduler_counts(&mut self, next: EngineLoadSnapshot) {
|
||||
self.last_scheduler_stats = Some(next);
|
||||
}
|
||||
}
|
||||
|
||||
/// Internal registry for tracking active requests and their output stream
|
||||
/// senders.
|
||||
///
|
||||
/// This is used to route incoming outputs to the correct request stream, and to
|
||||
/// ensure proper cleanup of senders when requests finish or the client shuts
|
||||
/// down.
|
||||
#[derive(Debug)]
|
||||
pub struct RequestRegistry {
|
||||
closed: bool,
|
||||
requests: BTreeMap<String, TrackedRequest>,
|
||||
routing_per_engine: BTreeMap<EngineId, EngineRoutingState>,
|
||||
}
|
||||
|
||||
impl RequestRegistry {
|
||||
pub fn new(engines: &[ConnectedEngine]) -> Self {
|
||||
Self {
|
||||
closed: false,
|
||||
requests: BTreeMap::default(),
|
||||
routing_per_engine: engines
|
||||
.iter()
|
||||
.map(|engine| (engine.engine_id.clone(), EngineRoutingState::default()))
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Register a newly added request. Create the per-request output channel
|
||||
/// bound to its `request_id` and return the selected engine id.
|
||||
///
|
||||
/// When `data_parallel_rank` is provided, the request is routed directly to
|
||||
/// the engine at that rank index, bypassing load balancing. Otherwise
|
||||
/// the engine with the fewest in-flight requests is chosen.
|
||||
pub fn register(
|
||||
&mut self,
|
||||
request_id: String,
|
||||
data_parallel_rank: Option<u32>,
|
||||
) -> Result<(EngineId, OutputReceiver)> {
|
||||
if self.requests.contains_key(&request_id) {
|
||||
return Err(Error::DuplicateRequestId { request_id });
|
||||
}
|
||||
|
||||
let engine_id = self.choose_engine_for_request(data_parallel_rank)?;
|
||||
let (tx, rx) = mpsc::unbounded_channel();
|
||||
self.requests.insert(
|
||||
request_id,
|
||||
TrackedRequest {
|
||||
sender: tx,
|
||||
engine_id: engine_id.clone(),
|
||||
},
|
||||
);
|
||||
|
||||
let state = self
|
||||
.routing_per_engine
|
||||
.get_mut(&engine_id)
|
||||
.expect("request registry must track all known engines");
|
||||
state.inflight += 1;
|
||||
|
||||
Ok((engine_id, rx))
|
||||
}
|
||||
|
||||
fn choose_engine_for_request(&mut self, data_parallel_rank: Option<u32>) -> Result<EngineId> {
|
||||
if let Some(rank) = data_parallel_rank {
|
||||
// Route to the engine at the specified rank index.
|
||||
let engine_id = EngineId::from_engine_index(rank);
|
||||
return self
|
||||
.routing_per_engine
|
||||
.contains_key(&engine_id)
|
||||
.then_some(engine_id)
|
||||
.ok_or_else(|| Error::InvalidDataParallelRank {
|
||||
rank,
|
||||
num_engines: self.routing_per_engine.len() as u32,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(self
|
||||
.routing_per_engine
|
||||
.iter()
|
||||
.min_by_key(|(_, state)| state.routing_score())
|
||||
.map(|(engine_id, _)| engine_id.clone())
|
||||
.expect("request registry must contain at least one engine"))
|
||||
}
|
||||
|
||||
/// Filter the given request IDs to the subset that are still tracked as
|
||||
/// active and can be aborted, grouped by engine.
|
||||
pub fn abortable_request_ids(&self, request_ids: &[String]) -> BTreeMap<EngineId, Vec<String>> {
|
||||
let mut by_engine = BTreeMap::new();
|
||||
for request_id in request_ids {
|
||||
let Some(tracked) = self.requests.get(request_id.as_str()) else {
|
||||
continue;
|
||||
};
|
||||
by_engine
|
||||
.entry(tracked.engine_id.clone())
|
||||
.or_insert_with(Vec::new)
|
||||
.push(request_id.clone());
|
||||
}
|
||||
by_engine
|
||||
}
|
||||
|
||||
/// Obtain the stream sender for one output. If it indicates the request is
|
||||
/// finished, it will be removed from the registry.
|
||||
pub fn sender_for_output(&mut self, output: &EngineCoreOutput) -> Option<OutputSender> {
|
||||
if output.finished() {
|
||||
self.remove(output.request_id.as_str()).map(|tracked| tracked.0)
|
||||
} else {
|
||||
self.requests
|
||||
.get(output.request_id.as_str())
|
||||
.map(|tracked| tracked.sender.clone())
|
||||
}
|
||||
}
|
||||
|
||||
/// Remove a batch of requests that have finished or aborted, returning
|
||||
/// their stream senders.
|
||||
pub fn finish_many<'a>(
|
||||
&mut self,
|
||||
request_ids: impl IntoIterator<Item = &'a String>,
|
||||
) -> Vec<OutputSender> {
|
||||
request_ids
|
||||
.into_iter()
|
||||
.filter_map(|request_id| self.remove(request_id.as_str()).map(|tracked| tracked.0))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Apply one scheduler stats update for the given engine to the local
|
||||
/// routing state. Returns `false` if the engine is unknown to the
|
||||
/// client.
|
||||
pub fn apply_scheduler_stats(&mut self, engine_index: u32, stats: &SchedulerStats) -> bool {
|
||||
self.apply_scheduler_counts(
|
||||
engine_index,
|
||||
EngineLoadSnapshot {
|
||||
waiting: stats.num_waiting_reqs as usize,
|
||||
running: stats.num_running_reqs as usize,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
/// Mark the registry as closed, detach and return all tracked senders.
|
||||
pub fn close(&mut self) -> Vec<OutputSender> {
|
||||
if self.closed {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
self.closed = true;
|
||||
std::mem::take(&mut self.requests)
|
||||
.into_values()
|
||||
.map(|tracked| tracked.sender)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Remove one request from the local registry. Returns the tracked entry if
|
||||
/// it exists.
|
||||
#[must_use]
|
||||
pub fn remove(&mut self, request_id: &str) -> Option<(OutputSender, EngineId)> {
|
||||
let tracked = self.requests.remove(request_id)?;
|
||||
self.routing_per_engine
|
||||
.get_mut(&tracked.engine_id)
|
||||
.expect("request registry must track all known engines")
|
||||
.inflight -= 1;
|
||||
Some((tracked.sender, tracked.engine_id))
|
||||
}
|
||||
|
||||
fn apply_scheduler_counts(&mut self, engine_index: u32, next: EngineLoadSnapshot) -> bool {
|
||||
let engine_id = EngineId::from_engine_index(engine_index);
|
||||
let Some(state) = self.routing_per_engine.get_mut(&engine_id) else {
|
||||
return false;
|
||||
};
|
||||
|
||||
let previous = state.last_scheduler_stats;
|
||||
if previous != Some(next) {
|
||||
trace!(
|
||||
?engine_id,
|
||||
previous_waiting = previous.map(|stats| stats.waiting),
|
||||
previous_running = previous.map(|stats| stats.running),
|
||||
waiting = next.waiting,
|
||||
running = next.running,
|
||||
"updated scheduler routing counts",
|
||||
);
|
||||
}
|
||||
|
||||
state.apply_scheduler_counts(next);
|
||||
true
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn contains(&self, request_id: &str) -> bool {
|
||||
self.requests.contains_key(request_id)
|
||||
}
|
||||
|
||||
pub fn is_closed(&self) -> bool {
|
||||
self.closed
|
||||
}
|
||||
}
|
||||
|
||||
/// Internal registry for tracking active utility calls and their waiting
|
||||
/// receivers.
|
||||
#[derive(Debug)]
|
||||
pub struct UtilityRegistry {
|
||||
closed: bool,
|
||||
next_call_id: AtomicI64,
|
||||
utility_calls: BTreeMap<i64, UtilitySender>,
|
||||
}
|
||||
|
||||
impl Default for UtilityRegistry {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
closed: false,
|
||||
next_call_id: AtomicI64::new(1),
|
||||
utility_calls: BTreeMap::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl UtilityRegistry {
|
||||
/// Allocate the next utility `call_id` and register a newly added utility
|
||||
/// call.
|
||||
pub fn allocate_and_register(&mut self) -> (i64, UtilityReceiver) {
|
||||
let call_id = self.next_call_id.fetch_add(1, Ordering::Relaxed);
|
||||
let (tx, rx) = oneshot::channel();
|
||||
self.utility_calls.insert(call_id, tx);
|
||||
(call_id, rx)
|
||||
}
|
||||
|
||||
/// Resolve a utility output to its waiting receiver.
|
||||
pub fn resolve(&mut self, call_id: &i64) -> Option<UtilitySender> {
|
||||
self.utility_calls.remove(call_id)
|
||||
}
|
||||
|
||||
/// Drop a batch of registered utility calls without delivering a result.
|
||||
/// Used to roll back allocations when the dispatch fan-out fails before
|
||||
/// every engine could accept the request.
|
||||
pub fn unregister_many(&mut self, call_ids: impl IntoIterator<Item = i64>) {
|
||||
for call_id in call_ids {
|
||||
self.utility_calls.remove(&call_id);
|
||||
}
|
||||
}
|
||||
|
||||
/// Mark the registry as closed, detach and return all tracked senders.
|
||||
pub fn close(&mut self) -> Vec<UtilitySender> {
|
||||
if self.closed {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
self.closed = true;
|
||||
std::mem::take(&mut self.utility_calls).into_values().collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn contains(&self, call_id: i64) -> bool {
|
||||
self.utility_calls.contains_key(&call_id)
|
||||
}
|
||||
|
||||
pub fn is_closed(&self) -> bool {
|
||||
self.closed
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{EngineRoutingState, RequestRegistry, UtilityRegistry};
|
||||
use crate::EngineId;
|
||||
use crate::client::state::EngineLoadSnapshot;
|
||||
use crate::protocol::{EngineCoreFinishReason, EngineCoreOutput};
|
||||
use crate::transport::ConnectedEngine;
|
||||
|
||||
#[test]
|
||||
fn registry_rejects_duplicate_request_ids() {
|
||||
let mut registry = RequestRegistry::new(&[ConnectedEngine {
|
||||
engine_id: EngineId::from(b"engine-0"),
|
||||
ready_response: None,
|
||||
}]);
|
||||
registry.register("req-1".to_string(), None).unwrap();
|
||||
let error = registry.register("req-1".to_string(), None).unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
crate::error::Error::DuplicateRequestId { request_id } if request_id == "req-1"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_removes_finished_request_on_output() {
|
||||
let mut registry = RequestRegistry::new(&[ConnectedEngine {
|
||||
engine_id: EngineId::from(b"engine-0"),
|
||||
ready_response: None,
|
||||
}]);
|
||||
registry.register("req-1".to_string(), None).unwrap();
|
||||
|
||||
let sender = registry.sender_for_output(&EngineCoreOutput {
|
||||
request_id: "req-1".to_string(),
|
||||
finish_reason: Some(EngineCoreFinishReason::Length),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
assert!(sender.is_some());
|
||||
assert!(!registry.contains("req-1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_closes_all_requests_on_failure() {
|
||||
let mut registry = RequestRegistry::new(&[ConnectedEngine {
|
||||
engine_id: EngineId::from(b"engine-0"),
|
||||
ready_response: None,
|
||||
}]);
|
||||
registry.register("req-1".to_string(), None).unwrap();
|
||||
registry.register("req-2".to_string(), None).unwrap();
|
||||
|
||||
let senders = registry.close();
|
||||
|
||||
assert_eq!(senders.len(), 2);
|
||||
assert!(registry.is_closed());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_tracks_engine_id_per_request() {
|
||||
let engine_0 = EngineId::from_engine_index(0);
|
||||
let engine_1 = EngineId::from_engine_index(1);
|
||||
let mut registry = RequestRegistry::new(&[
|
||||
ConnectedEngine {
|
||||
engine_id: engine_0.clone(),
|
||||
ready_response: None,
|
||||
},
|
||||
ConnectedEngine {
|
||||
engine_id: engine_1.clone(),
|
||||
ready_response: None,
|
||||
},
|
||||
]);
|
||||
let (chosen_0, _) = registry.register("req-1".to_string(), None).unwrap();
|
||||
let (chosen_1, _) = registry.register("req-2".to_string(), None).unwrap();
|
||||
let (chosen_0_again, _) = registry.register("req-3".to_string(), None).unwrap();
|
||||
|
||||
assert_eq!(chosen_0, engine_0);
|
||||
assert_eq!(chosen_1, engine_1);
|
||||
assert_eq!(chosen_0_again, engine_0);
|
||||
|
||||
let grouped = registry.abortable_request_ids(&[
|
||||
"req-1".to_string(),
|
||||
"req-2".to_string(),
|
||||
"req-3".to_string(),
|
||||
]);
|
||||
assert_eq!(
|
||||
grouped.get(&engine_0).unwrap(),
|
||||
&vec!["req-1".to_string(), "req-3".to_string()]
|
||||
);
|
||||
assert_eq!(grouped.get(&engine_1).unwrap(), &vec!["req-2".to_string()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_uses_inflight_as_waiting_fallback_before_stats_arrive() {
|
||||
let engine_0 = EngineId::from_engine_index(0);
|
||||
let engine_1 = EngineId::from_engine_index(1);
|
||||
let mut registry = RequestRegistry::new(&[
|
||||
ConnectedEngine {
|
||||
engine_id: engine_0.clone(),
|
||||
ready_response: None,
|
||||
},
|
||||
ConnectedEngine {
|
||||
engine_id: engine_1.clone(),
|
||||
ready_response: None,
|
||||
},
|
||||
]);
|
||||
|
||||
let (chosen_0, _) = registry.register("req-1".to_string(), None).unwrap();
|
||||
let (chosen_1, _) = registry.register("req-2".to_string(), None).unwrap();
|
||||
let (chosen_0_again, _) = registry.register("req-3".to_string(), None).unwrap();
|
||||
|
||||
assert_eq!(chosen_0, engine_0);
|
||||
assert_eq!(chosen_1, engine_1);
|
||||
assert_eq!(chosen_0_again, engine_0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn routing_score_uses_inflight_before_stats_arrive() {
|
||||
let state = EngineRoutingState {
|
||||
inflight: 3,
|
||||
last_scheduler_stats: None,
|
||||
};
|
||||
|
||||
assert_eq!(state.routing_score(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn routing_score_uses_inflight_as_scheduler_stats_lower_bound() {
|
||||
let state = EngineRoutingState {
|
||||
inflight: 7,
|
||||
last_scheduler_stats: Some(EngineLoadSnapshot {
|
||||
waiting: 0,
|
||||
running: 2,
|
||||
}),
|
||||
};
|
||||
|
||||
assert_eq!(state.routing_score(), 7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn routing_score_keeps_extra_waiting_penalty() {
|
||||
let state = EngineRoutingState {
|
||||
inflight: 1,
|
||||
last_scheduler_stats: Some(EngineLoadSnapshot {
|
||||
waiting: 3,
|
||||
running: 2,
|
||||
}),
|
||||
};
|
||||
|
||||
assert_eq!(state.routing_score(), 14);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_prefers_real_scheduler_stats_over_inflight() {
|
||||
let engine_0 = EngineId::from_engine_index(0);
|
||||
let engine_1 = EngineId::from_engine_index(1);
|
||||
let mut registry = RequestRegistry::new(&[
|
||||
ConnectedEngine {
|
||||
engine_id: engine_0.clone(),
|
||||
ready_response: None,
|
||||
},
|
||||
ConnectedEngine {
|
||||
engine_id: engine_1.clone(),
|
||||
ready_response: None,
|
||||
},
|
||||
]);
|
||||
|
||||
assert!(registry.apply_scheduler_counts(
|
||||
0,
|
||||
EngineLoadSnapshot {
|
||||
waiting: 3,
|
||||
running: 2
|
||||
}
|
||||
));
|
||||
assert!(registry.apply_scheduler_counts(
|
||||
1,
|
||||
EngineLoadSnapshot {
|
||||
waiting: 0,
|
||||
running: 1
|
||||
}
|
||||
));
|
||||
|
||||
let (chosen, _) = registry.register("req-stats".to_string(), None).unwrap();
|
||||
assert_eq!(chosen, engine_1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn register_with_data_parallel_rank_routes_to_specified_engine() {
|
||||
let engine_0 = EngineId::from_engine_index(0);
|
||||
let engine_1 = EngineId::from_engine_index(1);
|
||||
let engine_2 = EngineId::from_engine_index(2);
|
||||
let mut registry = RequestRegistry::new(&[
|
||||
ConnectedEngine {
|
||||
engine_id: engine_0.clone(),
|
||||
ready_response: None,
|
||||
},
|
||||
ConnectedEngine {
|
||||
engine_id: engine_1.clone(),
|
||||
ready_response: None,
|
||||
},
|
||||
ConnectedEngine {
|
||||
engine_id: engine_2.clone(),
|
||||
ready_response: None,
|
||||
},
|
||||
]);
|
||||
|
||||
// Explicitly target rank 2 (third engine).
|
||||
let (chosen, _) = registry.register("req-1".to_string(), Some(2)).unwrap();
|
||||
assert_eq!(chosen, engine_2);
|
||||
|
||||
// Explicitly target rank 0 (first engine).
|
||||
let (chosen, _) = registry.register("req-2".to_string(), Some(0)).unwrap();
|
||||
assert_eq!(chosen, engine_0);
|
||||
|
||||
// Explicitly target rank 1.
|
||||
let (chosen, _) = registry.register("req-3".to_string(), Some(1)).unwrap();
|
||||
assert_eq!(chosen, engine_1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn register_with_data_parallel_rank_bypasses_load_balancing() {
|
||||
let engine_0 = EngineId::from_engine_index(0);
|
||||
let engine_1 = EngineId::from_engine_index(1);
|
||||
let mut registry = RequestRegistry::new(&[
|
||||
ConnectedEngine {
|
||||
engine_id: engine_0.clone(),
|
||||
ready_response: None,
|
||||
},
|
||||
ConnectedEngine {
|
||||
engine_id: engine_1.clone(),
|
||||
ready_response: None,
|
||||
},
|
||||
]);
|
||||
|
||||
// Load-balance: first two go to engine_0 and engine_1.
|
||||
registry.register("req-lb-0".to_string(), None).unwrap();
|
||||
|
||||
// Now engine_0 has 1 in-flight. Without dp_rank, next would go to engine_1.
|
||||
// But with dp_rank=0, it should still go to engine_0.
|
||||
let (chosen, _) = registry.register("req-dp".to_string(), Some(0)).unwrap();
|
||||
assert_eq!(chosen, engine_0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn register_with_out_of_range_rank_returns_error() {
|
||||
let mut registry = RequestRegistry::new(&[
|
||||
ConnectedEngine {
|
||||
engine_id: EngineId::from_engine_index(0),
|
||||
ready_response: None,
|
||||
},
|
||||
ConnectedEngine {
|
||||
engine_id: EngineId::from_engine_index(1),
|
||||
ready_response: None,
|
||||
},
|
||||
]);
|
||||
|
||||
let error = registry.register("req-1".to_string(), Some(2)).unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
crate::error::Error::InvalidDataParallelRank {
|
||||
rank: 2,
|
||||
num_engines: 2,
|
||||
}
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn register_with_rank_on_single_engine_only_accepts_zero() {
|
||||
let engine_0 = EngineId::from_engine_index(0);
|
||||
let mut registry = RequestRegistry::new(&[ConnectedEngine {
|
||||
engine_id: engine_0.clone(),
|
||||
ready_response: None,
|
||||
}]);
|
||||
|
||||
let (chosen, _) = registry.register("req-ok".to_string(), Some(0)).unwrap();
|
||||
assert_eq!(chosen, engine_0);
|
||||
|
||||
let error = registry.register("req-bad".to_string(), Some(1)).unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
crate::error::Error::InvalidDataParallelRank {
|
||||
rank: 1,
|
||||
num_engines: 1,
|
||||
}
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn utility_registry_tracks_and_removes_call_ids() {
|
||||
let mut registry = UtilityRegistry::default();
|
||||
let (call_id_1, _) = registry.allocate_and_register();
|
||||
let (call_id_2, _) = registry.allocate_and_register();
|
||||
|
||||
assert_eq!(call_id_1, 1);
|
||||
assert_eq!(call_id_2, 2);
|
||||
assert!(registry.contains(1));
|
||||
assert!(registry.contains(2));
|
||||
assert!(registry.resolve(&1).is_some());
|
||||
assert!(!registry.contains(1));
|
||||
assert!(registry.contains(2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn utility_registry_closes_all_waiters_on_failure() {
|
||||
let mut registry = UtilityRegistry::default();
|
||||
registry.allocate_and_register();
|
||||
registry.allocate_and_register();
|
||||
|
||||
let senders = registry.close();
|
||||
|
||||
assert_eq!(senders.len(), 2);
|
||||
assert!(!registry.contains(1));
|
||||
assert!(!registry.contains(2));
|
||||
assert!(registry.is_closed());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn utility_registry_unregister_many_drops_pending_calls() {
|
||||
use tokio::sync::oneshot::error::TryRecvError;
|
||||
|
||||
let mut registry = UtilityRegistry::default();
|
||||
let (call_id_1, mut rx_1) = registry.allocate_and_register();
|
||||
let (call_id_2, mut rx_2) = registry.allocate_and_register();
|
||||
let (call_id_3, _rx_3) = registry.allocate_and_register();
|
||||
|
||||
// Drop two of the three allocated calls; the third stays pending.
|
||||
registry.unregister_many([call_id_1, call_id_2]);
|
||||
|
||||
assert!(!registry.contains(call_id_1));
|
||||
assert!(!registry.contains(call_id_2));
|
||||
assert!(registry.contains(call_id_3));
|
||||
// The receivers must observe the sender being dropped (channel closed).
|
||||
assert!(matches!(rx_1.try_recv(), Err(TryRecvError::Closed)));
|
||||
assert!(matches!(rx_2.try_recv(), Err(TryRecvError::Closed)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn utility_registry_unregister_many_ignores_unknown_call_ids() {
|
||||
let mut registry = UtilityRegistry::default();
|
||||
let (call_id, _rx) = registry.allocate_and_register();
|
||||
|
||||
// Unknown call ids are silently ignored — caller doesn't care which were live.
|
||||
registry.unregister_many([call_id, 42, 9999]);
|
||||
|
||||
assert!(!registry.contains(call_id));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
use std::ops::Deref;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
|
||||
use futures::Stream;
|
||||
use futures::stream::FusedStream;
|
||||
use thiserror_ext::AsReport as _;
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
use crate::client::AbortRequest;
|
||||
use crate::client::state::OutputReceiver;
|
||||
use crate::protocol::{EngineCoreFinishReason, EngineCoreOutput};
|
||||
use crate::{AbortCause, Error, Result};
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum State {
|
||||
Running,
|
||||
Finished,
|
||||
ClosedWithError,
|
||||
UnexpectedClose,
|
||||
}
|
||||
|
||||
/// One request-scoped engine-core output plus the enclosing batch metadata.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct EngineCoreStreamOutput {
|
||||
pub engine_index: u32,
|
||||
pub timestamp: f64,
|
||||
pub output: EngineCoreOutput,
|
||||
}
|
||||
|
||||
impl Deref for EngineCoreStreamOutput {
|
||||
type Target = EngineCoreOutput;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.output
|
||||
}
|
||||
}
|
||||
|
||||
/// Stream of raw engine-core outputs for one request.
|
||||
///
|
||||
/// The stream yields only [`EngineCoreStreamOutput`] values whose embedded
|
||||
/// output `request_id` matches the originating `add_request()` call. Normal
|
||||
/// request completion is expected to include a final output object whose
|
||||
/// `finish_reason` is non-`None`.
|
||||
pub struct EngineCoreOutputStream {
|
||||
request_id: String,
|
||||
abort_tx: mpsc::UnboundedSender<AbortRequest>,
|
||||
state: State,
|
||||
rx: OutputReceiver,
|
||||
}
|
||||
|
||||
impl EngineCoreOutputStream {
|
||||
pub(crate) fn new(
|
||||
request_id: String,
|
||||
abort_tx: mpsc::UnboundedSender<AbortRequest>,
|
||||
rx: OutputReceiver,
|
||||
) -> Self {
|
||||
Self {
|
||||
request_id,
|
||||
abort_tx,
|
||||
state: State::Running,
|
||||
rx,
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the engine-core `request_id` bound to this stream.
|
||||
pub fn request_id(&self) -> &str {
|
||||
&self.request_id
|
||||
}
|
||||
}
|
||||
|
||||
impl Stream for EngineCoreOutputStream {
|
||||
type Item = Result<EngineCoreStreamOutput>;
|
||||
|
||||
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
if self.is_terminated() {
|
||||
return Poll::Ready(None);
|
||||
}
|
||||
|
||||
match Pin::new(&mut self.rx).poll_recv(cx) {
|
||||
Poll::Pending => Poll::Pending,
|
||||
Poll::Ready(Some(item)) => {
|
||||
match &item {
|
||||
Ok(output) => {
|
||||
// If the output indicates the request is finished, mark the stream as
|
||||
// terminated with cleanly-finished state and expect no more outputs to
|
||||
// come.
|
||||
if output.finished() {
|
||||
if output.finish_reason == Some(EngineCoreFinishReason::Error) {
|
||||
error!(
|
||||
self.request_id,
|
||||
"request failed with an internal error during generation"
|
||||
);
|
||||
}
|
||||
debug!(self.request_id, "request completed via final output");
|
||||
self.state = State::Finished;
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
// If we get an error from the output stream, mark the stream as terminated
|
||||
// with an error.
|
||||
warn!(self.request_id, error = %error.as_report(), "request encountered an error");
|
||||
self.state = State::ClosedWithError;
|
||||
}
|
||||
}
|
||||
Poll::Ready(Some(item))
|
||||
}
|
||||
Poll::Ready(None) => {
|
||||
// If we get a `None` without seeing a finished output, this is an unexpected
|
||||
// close from the engine side. Mark the stream as terminated
|
||||
// with an unexpected close state and send an error down the
|
||||
// stream to notify the caller.
|
||||
warn!(self.request_id, "request stream closed unexpectedly");
|
||||
self.state = State::UnexpectedClose;
|
||||
|
||||
Poll::Ready(Some(Err(Error::RequestStreamClosed {
|
||||
request_id: self.request_id.clone(),
|
||||
})))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FusedStream for EngineCoreOutputStream {
|
||||
fn is_terminated(&self) -> bool {
|
||||
!matches!(self.state, State::Running)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for EngineCoreOutputStream {
|
||||
fn drop(&mut self) {
|
||||
if self.is_terminated() {
|
||||
// If it's terminated, it means that the request either finished cleanly, or
|
||||
// encountered an error or unexpected close from the engine. In any
|
||||
// case, the request stream is already considered inactive and
|
||||
// there's no need to abort it on the engine side.
|
||||
return;
|
||||
}
|
||||
|
||||
let abort_req = AbortRequest {
|
||||
request_id: self.request_id.clone(),
|
||||
cause: AbortCause::current(),
|
||||
};
|
||||
|
||||
if self.abort_tx.send(abort_req).is_err() {
|
||||
warn!(
|
||||
request_id = self.request_id,
|
||||
"auto-abort worker already shut down; skip auto-abort"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use bytes::Bytes;
|
||||
use zeromq::prelude::{Socket, SocketRecv, SocketSend};
|
||||
use zeromq::{PullSocket, XPubSocket, ZmqMessage};
|
||||
|
||||
use crate::error::{Error, Result, bail_unexpected_handshake_message};
|
||||
|
||||
/// Engine-facing sockets owned by the in-process coordinator.
|
||||
pub(crate) struct CoordinatorBootstrap {
|
||||
pub input_address: String,
|
||||
pub output_address: String,
|
||||
pub input_socket: XPubSocket,
|
||||
pub output_socket: PullSocket,
|
||||
}
|
||||
|
||||
impl CoordinatorBootstrap {
|
||||
/// Bind the engine-facing coordinator sockets on the given host.
|
||||
pub(crate) async fn bind(local_host: &str) -> Result<Self> {
|
||||
let mut input_socket = XPubSocket::new();
|
||||
let input_address = input_socket.bind(&format!("tcp://{local_host}:0")).await?.to_string();
|
||||
|
||||
let mut output_socket = PullSocket::new();
|
||||
let output_address =
|
||||
output_socket.bind(&format!("tcp://{local_host}:0")).await?.to_string();
|
||||
|
||||
Ok(Self {
|
||||
input_address,
|
||||
output_address,
|
||||
input_socket,
|
||||
output_socket,
|
||||
})
|
||||
}
|
||||
|
||||
/// Complete the engine-facing startup gate before engines are allowed to
|
||||
/// send handshake READY.
|
||||
pub(crate) async fn wait_for_startup_gate(
|
||||
&mut self,
|
||||
engine_count: usize,
|
||||
ready_timeout: Duration,
|
||||
) -> Result<()> {
|
||||
wait_for_engine_subscriptions(&mut self.input_socket, engine_count, ready_timeout).await?;
|
||||
send_ready_to_engines(&mut self.input_socket).await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Wait until all engines subscribe to the coordinator broadcast socket.
|
||||
async fn wait_for_engine_subscriptions(
|
||||
input_socket: &mut XPubSocket,
|
||||
engine_count: usize,
|
||||
ready_timeout: Duration,
|
||||
) -> Result<()> {
|
||||
let mut received = 0;
|
||||
while received < engine_count {
|
||||
let message =
|
||||
tokio::time::timeout(ready_timeout, input_socket.recv()).await.map_err(|_| {
|
||||
Error::HandshakeTimeout {
|
||||
stage: "coordinator engine subscriptions",
|
||||
timeout: ready_timeout,
|
||||
}
|
||||
})??;
|
||||
if message.len() != 1 {
|
||||
bail_unexpected_handshake_message!(
|
||||
"expected 1 frame for coordinator subscription, got {}",
|
||||
message.len()
|
||||
);
|
||||
}
|
||||
|
||||
let frame = message
|
||||
.into_vec()
|
||||
.into_iter()
|
||||
.next()
|
||||
.expect("single-frame coordinator subscription message");
|
||||
if frame.as_ref() != [0x01] {
|
||||
bail_unexpected_handshake_message!(
|
||||
"expected coordinator subscription frame [0x01], got {:?}",
|
||||
frame.as_ref()
|
||||
);
|
||||
}
|
||||
received += 1;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Send the coordinator READY marker to all subscribed engines.
|
||||
async fn send_ready_to_engines(input_socket: &mut XPubSocket) -> Result<()> {
|
||||
input_socket.send(ZmqMessage::from(Bytes::from_static(b"READY"))).await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
|
||||
use thiserror_ext::AsReport;
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::{debug, warn};
|
||||
use zeromq::prelude::{SocketRecv, SocketSend};
|
||||
use zeromq::{XSubSocket, ZmqMessage};
|
||||
|
||||
use crate::client::imp::ClientInner;
|
||||
use crate::coordinator::handle::{CoordinatorCommand, CoordinatorState};
|
||||
use crate::error::{Error, Result, bail_unexpected_coordinator_output};
|
||||
use crate::protocol::{OpaqueValue, decode_msgpack, encode_msgpack};
|
||||
|
||||
/// Frontend-to-coordinator wakeup message sent when the first request arrives
|
||||
/// while all engines are paused.
|
||||
///
|
||||
/// This matches the frontend-side msgpack tuple sent by Python
|
||||
/// `DPAsyncMPClient._ensure_stats_update_task` to the coordinator front socket.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/694449050f8dac3d9853e97e518b4a43ec52106a/vllm/v1/engine/core_client.py#L1230-L1236>
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize_tuple, Deserialize_tuple)]
|
||||
struct CoordinatorWakeupMessage {
|
||||
/// Engine index that already has the triggering request and should be
|
||||
/// excluded from the coordinator's `START_DP_WAVE` rebroadcast.
|
||||
exclude_engine_index: u32,
|
||||
/// DP wave number observed by the frontend when the request was admitted.
|
||||
wave: u32,
|
||||
}
|
||||
|
||||
/// Coordinator-to-frontend state publish received on the front-side coordinator
|
||||
/// socket.
|
||||
///
|
||||
/// This matches the msgpack tuple periodically published by Python
|
||||
/// `DPCoordinatorProc.run_coordinator` to all connected frontends.
|
||||
///
|
||||
/// Original Python definitions:
|
||||
/// <https://github.com/vllm-project/vllm/blob/694449050f8dac3d9853e97e518b4a43ec52106a/vllm/v1/engine/coordinator.py#L282-L283>
|
||||
/// <https://github.com/vllm-project/vllm/blob/694449050f8dac3d9853e97e518b4a43ec52106a/vllm/v1/engine/coordinator.py#L445-L447>
|
||||
#[derive(Debug, Clone, PartialEq, Deserialize_tuple)]
|
||||
struct CoordinatorStateUpdate {
|
||||
/// Global per-engine request counts published by the coordinator.
|
||||
///
|
||||
/// The Rust bootstrapped external-coordinator path preserves this field for
|
||||
/// wire compatibility but intentionally ignores it for routing decisions.
|
||||
counts: OpaqueValue,
|
||||
/// Current global DP wave number stamped onto newly admitted requests.
|
||||
wave: u32,
|
||||
/// Whether engines are currently running (`true`) or paused (`false`).
|
||||
engines_running: bool,
|
||||
}
|
||||
|
||||
/// Background half of an external Python-owned coordinator connection.
|
||||
///
|
||||
/// This owns the command receiver and one frontend-facing XSUB socket. It
|
||||
/// mirrors the subset of Python's coordinator protocol needed by the Rust
|
||||
/// bootstrapped frontend: receive `(counts, wave, running)` publishes, ignore
|
||||
/// `counts`, and send `(exclude_engine_index, wave)` wakeup messages when the
|
||||
/// first request arrives while engines are paused.
|
||||
pub(crate) struct ExternalCoordinatorService {
|
||||
state: Arc<CoordinatorState>,
|
||||
command_rx: mpsc::UnboundedReceiver<CoordinatorCommand>,
|
||||
socket: XSubSocket,
|
||||
}
|
||||
|
||||
impl ExternalCoordinatorService {
|
||||
pub(super) fn new(
|
||||
state: Arc<CoordinatorState>,
|
||||
command_rx: mpsc::UnboundedReceiver<CoordinatorCommand>,
|
||||
socket: XSubSocket,
|
||||
) -> Self {
|
||||
Self {
|
||||
state,
|
||||
command_rx,
|
||||
socket,
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply one frontend-originated command to the external coordinator state
|
||||
/// machine.
|
||||
async fn handle_command(&mut self, command: CoordinatorCommand) -> Result<()> {
|
||||
match command {
|
||||
CoordinatorCommand::FirstRequest {
|
||||
target_engine_id,
|
||||
wave,
|
||||
} => {
|
||||
let target_engine_index = target_engine_id.engine_index().ok_or_else(|| {
|
||||
Error::UnsupportedCoordinatorEngineId {
|
||||
engine_id: target_engine_id.to_vec(),
|
||||
}
|
||||
})?;
|
||||
debug!(
|
||||
wave,
|
||||
exclude_engine_index = target_engine_index,
|
||||
"notifying external coordinator about first request while engines were paused"
|
||||
);
|
||||
let payload = encode_msgpack(&CoordinatorWakeupMessage {
|
||||
exclude_engine_index: target_engine_index,
|
||||
wave,
|
||||
})?;
|
||||
self.socket.send(ZmqMessage::from(payload)).await?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Apply one publish received from the xsub socket containing a coordinator
|
||||
/// state update.
|
||||
async fn handle_publish(&mut self, message: ZmqMessage) -> Result<()> {
|
||||
let frames = message.into_vec();
|
||||
if frames.len() != 1 {
|
||||
bail_unexpected_coordinator_output!(
|
||||
"received malformed external coordinator publish with {} frame(s)",
|
||||
frames.len()
|
||||
);
|
||||
}
|
||||
|
||||
let update: CoordinatorStateUpdate = decode_msgpack(&frames[0])?;
|
||||
|
||||
let mut state = self.state.lock();
|
||||
let previous_wave = state.current_wave;
|
||||
let previous_engines_running = state.engines_running;
|
||||
state.current_wave = update.wave;
|
||||
state.engines_running = update.engines_running;
|
||||
debug!(
|
||||
previous_wave,
|
||||
wave = update.wave,
|
||||
previous_engines_running,
|
||||
engines_running = update.engines_running,
|
||||
"applied external coordinator state update"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Drive the coordinator event loop until either side of the control plane
|
||||
/// is closed or a fatal error is observed.
|
||||
pub(crate) async fn run(mut self, inner: Arc<ClientInner>) {
|
||||
let result: Result<()> = async {
|
||||
loop {
|
||||
tokio::select! {
|
||||
// Received frontend-originated command from the handle.
|
||||
command = self.command_rx.recv() => {
|
||||
let Some(command) = command else {
|
||||
warn!("external coordinator command channel closed, shutting down service");
|
||||
return Ok(());
|
||||
};
|
||||
self.handle_command(command).await?;
|
||||
}
|
||||
// Received publish from the external coordinator socket.
|
||||
publish = self.socket.recv() => {
|
||||
let publish = publish.map_err(Error::from)?;
|
||||
self.handle_publish(publish).await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
.await;
|
||||
let Err(error) = result else { return };
|
||||
|
||||
warn!(
|
||||
error = %error.as_report(),
|
||||
"external coordinator service exiting with error"
|
||||
);
|
||||
inner.close_registries(Arc::new(error));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use parking_lot::Mutex;
|
||||
use tokio::sync::mpsc;
|
||||
use zeromq::prelude::Socket;
|
||||
use zeromq::{XPubSocket, XSubSocket};
|
||||
|
||||
use crate::coordinator::external::ExternalCoordinatorService;
|
||||
use crate::coordinator::inproc::InProcCoordinatorRunner;
|
||||
use crate::error::{Error, Result, bail_control_closed};
|
||||
use crate::transport::EngineId;
|
||||
|
||||
/// Snapshot to the coordinator state for request routing and stamping.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) struct CoordinatorStateSnapshot {
|
||||
/// The current DP wave, which will be stamped on outgoing requests.
|
||||
pub current_wave: u32,
|
||||
/// Whether the engines are currently running or paused, which determines if
|
||||
/// the frontend must trigger a new wave on the next request.
|
||||
pub engines_running: bool,
|
||||
}
|
||||
|
||||
/// Shared in-process coordinator state.
|
||||
pub(crate) type CoordinatorState = Mutex<CoordinatorStateSnapshot>;
|
||||
|
||||
/// Commands sent from the frontend request path into the background runner.
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum CoordinatorCommand {
|
||||
/// The first request arrived while all engines were paused.
|
||||
///
|
||||
/// The coordinator should broadcast `START_DP_WAVE` with the current wave
|
||||
/// and the target engine index as the excluded engine.
|
||||
FirstRequest {
|
||||
target_engine_id: EngineId,
|
||||
wave: u32,
|
||||
},
|
||||
}
|
||||
|
||||
/// Frontend-facing coordinator handle used by `EngineCoreClient::call()`.
|
||||
///
|
||||
/// This side stays intentionally small: it can read the latest wave snapshot
|
||||
/// and enqueue a `FirstRequest` transition when the request path observes the
|
||||
/// system in the paused state.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct CoordinatorHandle {
|
||||
state: Arc<CoordinatorState>,
|
||||
command_tx: mpsc::UnboundedSender<CoordinatorCommand>,
|
||||
}
|
||||
|
||||
impl CoordinatorHandle {
|
||||
fn new_parts() -> (
|
||||
Self,
|
||||
Arc<CoordinatorState>,
|
||||
mpsc::UnboundedReceiver<CoordinatorCommand>,
|
||||
) {
|
||||
let state = Arc::new(Mutex::new(CoordinatorStateSnapshot {
|
||||
current_wave: 0,
|
||||
engines_running: false,
|
||||
}));
|
||||
let (command_tx, command_rx) = mpsc::unbounded_channel();
|
||||
(
|
||||
Self {
|
||||
state: state.clone(),
|
||||
command_tx,
|
||||
},
|
||||
state,
|
||||
command_rx,
|
||||
)
|
||||
}
|
||||
|
||||
/// Build the paired frontend handle and background runner around one
|
||||
/// engine-facing coordinator broadcast socket.
|
||||
pub(crate) fn new_inproc(coordinator_input: XPubSocket) -> (Self, InProcCoordinatorRunner) {
|
||||
let (handle, state, command_rx) = Self::new_parts();
|
||||
(
|
||||
handle,
|
||||
InProcCoordinatorRunner::new(state, command_rx, coordinator_input),
|
||||
)
|
||||
}
|
||||
|
||||
/// Build the paired frontend handle and background service around an
|
||||
/// external Python-owned frontend-side coordinator socket.
|
||||
pub(crate) async fn connect_external(
|
||||
coordinator_address: &str,
|
||||
) -> Result<(Self, ExternalCoordinatorService)> {
|
||||
let (handle, state, command_rx) = Self::new_parts();
|
||||
let mut socket = XSubSocket::new();
|
||||
socket.connect(coordinator_address).await?;
|
||||
socket.subscribe("").await?;
|
||||
Ok((
|
||||
handle,
|
||||
ExternalCoordinatorService::new(state, command_rx, socket),
|
||||
))
|
||||
}
|
||||
|
||||
/// Snapshot the coordinator state for request routing and stamping.
|
||||
pub(crate) fn snapshot(&self) -> CoordinatorStateSnapshot {
|
||||
*self.state.lock()
|
||||
}
|
||||
|
||||
/// Notify the runner that a new request arrived while engines were paused.
|
||||
///
|
||||
/// The handle flips `engines_running` optimistically so concurrent request
|
||||
/// submissions coalesce behind one `START_DP_WAVE` broadcast instead of all
|
||||
/// trying to trigger the wave independently.
|
||||
pub(crate) fn notify_first_request(&self, target_engine_id: EngineId) -> Result<()> {
|
||||
let mut state = self.state.lock();
|
||||
if state.engines_running {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let command = CoordinatorCommand::FirstRequest {
|
||||
target_engine_id,
|
||||
wave: state.current_wave,
|
||||
};
|
||||
if self.command_tx.send(command).is_err() {
|
||||
bail_control_closed!("in-process coordinator command channel already shut down");
|
||||
}
|
||||
|
||||
state.engines_running = true;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
|
||||
use thiserror_ext::AsReport;
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::{debug, warn};
|
||||
use zeromq::prelude::SocketSend;
|
||||
use zeromq::{XPubSocket, ZmqMessage};
|
||||
|
||||
use crate::client::imp::ClientInner;
|
||||
use crate::coordinator::handle::{CoordinatorCommand, CoordinatorState};
|
||||
use crate::error::{Error, Result, bail_unexpected_coordinator_output};
|
||||
use crate::protocol::{
|
||||
ClassifiedEngineCoreOutputs, DpControlMessage, EngineCoreOutputs, EngineCoreRequestType,
|
||||
encode_msgpack,
|
||||
};
|
||||
|
||||
/// Coordinator-to-engine `START_DP_WAVE` control payload encoded on the
|
||||
/// engine-facing coordinator socket.
|
||||
///
|
||||
/// This matches the msgpack tuple broadcast by Python
|
||||
/// `DPCoordinatorProc._send_start_wave`.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/694449050f8dac3d9853e97e518b4a43ec52106a/vllm/v1/engine/coordinator.py#L453-L459>
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize_tuple, Deserialize_tuple)]
|
||||
struct StartDpWaveMessage {
|
||||
/// DP wave number that all engines should start processing.
|
||||
wave: u32,
|
||||
/// Engine index that already received the triggering request and should not
|
||||
/// receive an extra wakeup notification.
|
||||
exclude_engine_index: u32,
|
||||
}
|
||||
|
||||
/// Background half of the in-process coordinator.
|
||||
///
|
||||
/// This owns the command receiver and the engine-facing coordinator input
|
||||
/// socket. It is the single place where wave transitions are serialized and
|
||||
/// where `START_DP_WAVE` broadcasts are emitted.
|
||||
pub(crate) struct InProcCoordinatorRunner {
|
||||
state: Arc<CoordinatorState>,
|
||||
command_rx: mpsc::UnboundedReceiver<CoordinatorCommand>,
|
||||
coordinator_input: XPubSocket,
|
||||
}
|
||||
|
||||
impl InProcCoordinatorRunner {
|
||||
pub(super) fn new(
|
||||
state: Arc<CoordinatorState>,
|
||||
command_rx: mpsc::UnboundedReceiver<CoordinatorCommand>,
|
||||
coordinator_input: XPubSocket,
|
||||
) -> Self {
|
||||
Self {
|
||||
state,
|
||||
command_rx,
|
||||
coordinator_input,
|
||||
}
|
||||
}
|
||||
|
||||
/// Broadcast Python-compatible `START_DP_WAVE` to all connected engines.
|
||||
async fn broadcast_start_wave(&mut self, wave: u32, exclude_engine_index: u32) -> Result<()> {
|
||||
let payload = encode_msgpack(&StartDpWaveMessage {
|
||||
wave,
|
||||
exclude_engine_index,
|
||||
})?;
|
||||
self.coordinator_input
|
||||
.send(
|
||||
ZmqMessage::try_from(vec![
|
||||
EngineCoreRequestType::StartDpWave.to_frame(),
|
||||
payload.into(),
|
||||
])
|
||||
.expect("coordinator START_DP_WAVE message must contain two frames"),
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Apply one frontend-originated command to the coordinator state machine.
|
||||
async fn handle_command(&mut self, command: CoordinatorCommand) -> Result<()> {
|
||||
match command {
|
||||
CoordinatorCommand::FirstRequest {
|
||||
target_engine_id,
|
||||
wave,
|
||||
} => {
|
||||
let target_engine_index = target_engine_id.engine_index().ok_or_else(|| {
|
||||
Error::UnsupportedCoordinatorEngineId {
|
||||
engine_id: target_engine_id.to_vec(),
|
||||
}
|
||||
})?;
|
||||
self.state.lock().current_wave = wave;
|
||||
debug!(
|
||||
wave,
|
||||
exclude_engine_index = target_engine_index,
|
||||
"starting DP wave after first request while engines were paused"
|
||||
);
|
||||
self.broadcast_start_wave(wave, target_engine_index).await?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Apply one engine-originated control output to the coordinator state
|
||||
/// machine.
|
||||
async fn handle_outputs(&mut self, outputs: EngineCoreOutputs) -> Result<()> {
|
||||
match outputs.classify() {
|
||||
ClassifiedEngineCoreOutputs::RequestBatch(batch)
|
||||
if batch.outputs.is_empty() && batch.finished_requests.is_none() =>
|
||||
{
|
||||
// Stats-only output for coordinator.
|
||||
// Ignore since the Rust coordinator doesn't track stats for
|
||||
// routing decisions.
|
||||
}
|
||||
ClassifiedEngineCoreOutputs::DpControl {
|
||||
engine_index,
|
||||
control,
|
||||
..
|
||||
} => match control {
|
||||
// The engines signals they completed the current wave and are now paused.
|
||||
// Advance the current wave and mark the state as paused.
|
||||
DpControlMessage::WaveComplete(wave) => {
|
||||
let mut state = self.state.lock();
|
||||
if wave >= state.current_wave {
|
||||
let next_wave = wave + 1;
|
||||
debug!(
|
||||
wave,
|
||||
next_wave,
|
||||
"DP wave finished; pausing engines and advancing coordinator state"
|
||||
);
|
||||
state.current_wave = wave + 1;
|
||||
state.engines_running = false;
|
||||
}
|
||||
}
|
||||
// An engine requests to start the wave.
|
||||
// Rebroadcast the wave to all engines except for the originated one.
|
||||
DpControlMessage::StartWave(wave) => {
|
||||
let should_broadcast = {
|
||||
let mut state = self.state.lock();
|
||||
if wave > state.current_wave
|
||||
|| (wave == state.current_wave && !state.engines_running)
|
||||
{
|
||||
state.current_wave = wave;
|
||||
state.engines_running = true;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
};
|
||||
if should_broadcast {
|
||||
debug!(
|
||||
wave,
|
||||
exclude_engine_index = engine_index,
|
||||
"starting DP wave after stale-wave notification from engine"
|
||||
);
|
||||
self.broadcast_start_wave(wave, engine_index).await?;
|
||||
}
|
||||
}
|
||||
},
|
||||
other => {
|
||||
bail_unexpected_coordinator_output!(
|
||||
"received non-control output on coordinator path: {other:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Drive the coordinator event loop until either side of the control plane
|
||||
/// is closed or a fatal error is observed.
|
||||
///
|
||||
/// Any fatal error closes the main client registries so request streams and
|
||||
/// future calls observe a stable shutdown cause.
|
||||
pub(crate) async fn run(
|
||||
mut self,
|
||||
mut output_rx: mpsc::Receiver<Result<EngineCoreOutputs>>,
|
||||
inner: Arc<ClientInner>,
|
||||
) {
|
||||
let result: Result<()> = async {
|
||||
loop {
|
||||
tokio::select! {
|
||||
// Received frontend-originated command from the handle.
|
||||
command = self.command_rx.recv() => {
|
||||
let Some(command) = command else {
|
||||
warn!("coordinator command channel closed, shutting down coordinator runner");
|
||||
return Ok(());
|
||||
};
|
||||
self.handle_command(command).await?;
|
||||
}
|
||||
// Received engine-originated control output from the coordinator socket.
|
||||
outputs = output_rx.recv() => {
|
||||
let Some(outputs) = outputs else {
|
||||
warn!("coordinator output channel closed, shutting down coordinator runner");
|
||||
return Ok(());
|
||||
};
|
||||
self.handle_outputs(outputs?).await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
.await;
|
||||
let Err(error) = result else { return };
|
||||
|
||||
warn!(error = %error.as_report(), "coordinator runner exiting with error");
|
||||
inner.close_registries(Arc::new(error));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
mod bootstrap;
|
||||
mod external;
|
||||
mod handle;
|
||||
mod inproc;
|
||||
|
||||
pub(crate) use bootstrap::CoordinatorBootstrap;
|
||||
pub(crate) use handle::CoordinatorHandle;
|
||||
@@ -0,0 +1,88 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use thiserror::Error;
|
||||
use thiserror_ext::Macro;
|
||||
|
||||
pub type Result<T> = std::result::Result<T, Error>;
|
||||
|
||||
/// Public error type for the Rust engine-core client.
|
||||
#[derive(Debug, Error, Macro)]
|
||||
pub enum Error {
|
||||
#[error("messagepack encode failed for {target_type}: {message}")]
|
||||
Encode {
|
||||
target_type: &'static str,
|
||||
message: String,
|
||||
},
|
||||
#[error("messagepack decode failed for {target_type}: {message}")]
|
||||
Decode {
|
||||
target_type: &'static str,
|
||||
message: String,
|
||||
},
|
||||
#[error("messagepack value decode failed")]
|
||||
ValueDecode(#[from] rmpv::decode::Error),
|
||||
#[error("messagepack ext value decode failed: {message}")]
|
||||
ExtValueDecode { message: String },
|
||||
#[error("io error")]
|
||||
Io(#[from] std::io::Error),
|
||||
#[error("transport error")]
|
||||
Transport(#[from] zeromq::ZmqError),
|
||||
#[error("engine core reported fatal failure")]
|
||||
EngineCoreDead,
|
||||
#[error("startup handshake timed out while waiting for {stage} after {timeout:?}")]
|
||||
HandshakeTimeout {
|
||||
stage: &'static str,
|
||||
timeout: Duration,
|
||||
},
|
||||
#[error("engine input registration timed out after {timeout:?}")]
|
||||
InputRegistrationTimeout { timeout: Duration },
|
||||
#[error("unexpected engine id in startup handshake: expected {expected:?}, got {actual:?}")]
|
||||
UnexpectedHandshakeIdentity { expected: Vec<u8>, actual: Vec<u8> },
|
||||
#[error("unexpected startup handshake message: {message}")]
|
||||
UnexpectedHandshakeMessage { message: String },
|
||||
#[error("unexpected non-control output on coordinator path: {message}")]
|
||||
UnexpectedCoordinatorOutput { message: String },
|
||||
#[error("unexpected output on main dispatcher path: {message}")]
|
||||
UnexpectedDispatcherOutput { message: String },
|
||||
#[error("coordinator requires a Python-compatible two-byte engine id, got {engine_id:?}")]
|
||||
UnsupportedCoordinatorEngineId { engine_id: Vec<u8> },
|
||||
#[error("unsupported auxiliary frame(s): expected 1 frame, got {frame_count}")]
|
||||
UnsupportedAuxFrames { frame_count: usize },
|
||||
#[error("external coordinator mode is not implemented yet")]
|
||||
UnsupportedExternalCoordinator,
|
||||
#[error("unsupported field `{field}` in {context}")]
|
||||
UnsupportedField {
|
||||
context: &'static str,
|
||||
field: &'static str,
|
||||
},
|
||||
#[error("engine control channel closed unexpectedly: {message}")]
|
||||
ControlClosed { message: String },
|
||||
#[error("request `{request_id}` is already in flight")]
|
||||
DuplicateRequestId { request_id: String },
|
||||
#[error("data parallel rank {rank} is out of range for {num_engines} engine(s)")]
|
||||
InvalidDataParallelRank { rank: u32, num_engines: u32 },
|
||||
#[error("engine-core output dispatcher closed: {message}")]
|
||||
DispatcherClosed { message: String },
|
||||
#[error("engine-core client is closed: {message}")]
|
||||
ClientClosed { message: String },
|
||||
#[error("request output stream for `{request_id}` closed unexpectedly")]
|
||||
RequestStreamClosed { request_id: String },
|
||||
#[error("utility call `{method}` failed (call_id={call_id}): {message}")]
|
||||
UtilityCallFailed {
|
||||
method: String,
|
||||
call_id: i64,
|
||||
message: String,
|
||||
},
|
||||
#[error("utility call `{method}` returned an invalid result (call_id={call_id}): {message}")]
|
||||
UtilityResultDecode {
|
||||
method: String,
|
||||
call_id: i64,
|
||||
message: String,
|
||||
},
|
||||
#[error("utility call `{method}` closed unexpectedly (call_id={call_id})")]
|
||||
UtilityCallClosed { method: String, call_id: i64 },
|
||||
|
||||
/// A special variant to allow cloning the same error.
|
||||
#[error(transparent)]
|
||||
Shared(Arc<Self>),
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
mod client;
|
||||
mod coordinator;
|
||||
mod error;
|
||||
mod metrics;
|
||||
pub mod protocol;
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub mod test_utils;
|
||||
mod transport;
|
||||
|
||||
pub use client::{
|
||||
AbortCause, CoordinatorMode, EngineCoreClient, EngineCoreClientConfig, EngineCoreOutputStream,
|
||||
EngineCoreStreamOutput, TransportMode,
|
||||
};
|
||||
pub use error::{Error, Result};
|
||||
pub use transport::{ENGINE_CORE_DEAD_SENTINEL, EngineId};
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,131 @@
|
||||
use vllm_metrics::{EngineLabels, EnginePositionLabels, SchedulerMetrics, WaitingReasonLabels};
|
||||
|
||||
use crate::protocol::stats::SchedulerStats;
|
||||
|
||||
const WAITING_REASON_CAPACITY: &str = "capacity";
|
||||
const WAITING_REASON_DEFERRED: &str = "deferred";
|
||||
|
||||
/// Record the scheduler-stats-backed metrics for one engine at one point in
|
||||
/// time.
|
||||
pub(crate) fn record_scheduler_stats(
|
||||
metrics: &SchedulerMetrics,
|
||||
model_name: impl Into<String>,
|
||||
engine: u32,
|
||||
stats: &SchedulerStats,
|
||||
) {
|
||||
let model_name = model_name.into();
|
||||
let labels = EngineLabels {
|
||||
model_name: model_name.clone(),
|
||||
engine,
|
||||
};
|
||||
|
||||
// Scheduler state gauges.
|
||||
metrics.scheduler_running.get_or_create(&labels).set(stats.num_running_reqs);
|
||||
metrics
|
||||
.scheduler_waiting
|
||||
.get_or_create(&labels)
|
||||
.set(stats.num_waiting_reqs + stats.num_skipped_waiting_reqs);
|
||||
metrics
|
||||
.scheduler_waiting_by_reason
|
||||
.get_or_create(&WaitingReasonLabels {
|
||||
model_name: model_name.clone(),
|
||||
engine,
|
||||
reason: WAITING_REASON_CAPACITY,
|
||||
})
|
||||
.set(stats.num_waiting_reqs);
|
||||
metrics
|
||||
.scheduler_waiting_by_reason
|
||||
.get_or_create(&WaitingReasonLabels {
|
||||
model_name: model_name.clone(),
|
||||
engine,
|
||||
reason: WAITING_REASON_DEFERRED,
|
||||
})
|
||||
.set(stats.num_skipped_waiting_reqs);
|
||||
metrics.kv_cache_usage.get_or_create(&labels).set(stats.kv_cache_usage);
|
||||
|
||||
// Prefix-cache counters, including the connector-backed external cache path.
|
||||
metrics
|
||||
.prefix_cache_queries
|
||||
.get_or_create(&labels)
|
||||
.inc_by(stats.prefix_cache_stats.base.queries);
|
||||
metrics
|
||||
.prefix_cache_hits
|
||||
.get_or_create(&labels)
|
||||
.inc_by(stats.prefix_cache_stats.base.hits);
|
||||
|
||||
if let Some(connector_prefix_cache_stats) = &stats.connector_prefix_cache_stats {
|
||||
metrics
|
||||
.external_prefix_cache_queries
|
||||
.get_or_create(&labels)
|
||||
.inc_by(connector_prefix_cache_stats.base.queries);
|
||||
metrics
|
||||
.external_prefix_cache_hits
|
||||
.get_or_create(&labels)
|
||||
.inc_by(connector_prefix_cache_stats.base.hits);
|
||||
}
|
||||
|
||||
// Speculative decoding counters.
|
||||
if let Some(spec_decoding_stats) = &stats.spec_decoding_stats {
|
||||
metrics
|
||||
.spec_decode_num_drafts
|
||||
.get_or_create(&labels)
|
||||
.inc_by(spec_decoding_stats.num_drafts);
|
||||
metrics
|
||||
.spec_decode_num_draft_tokens
|
||||
.get_or_create(&labels)
|
||||
.inc_by(spec_decoding_stats.num_draft_tokens);
|
||||
metrics
|
||||
.spec_decode_num_accepted_tokens
|
||||
.get_or_create(&labels)
|
||||
.inc_by(spec_decoding_stats.num_accepted_tokens);
|
||||
|
||||
for (position, accepted_tokens) in
|
||||
spec_decoding_stats.num_accepted_tokens_per_pos.iter().copied().enumerate()
|
||||
{
|
||||
metrics
|
||||
.spec_decode_num_accepted_tokens_per_pos
|
||||
.get_or_create(&EnginePositionLabels {
|
||||
model_name: model_name.clone(),
|
||||
engine,
|
||||
position: position as u32,
|
||||
})
|
||||
.inc_by(accepted_tokens);
|
||||
}
|
||||
}
|
||||
|
||||
// Per-engine performance / MFU counters.
|
||||
if let Some(perf_stats) = &stats.perf_stats
|
||||
&& (perf_stats.num_flops_per_gpu != 0
|
||||
|| perf_stats.num_read_bytes_per_gpu != 0
|
||||
|| perf_stats.num_write_bytes_per_gpu != 0)
|
||||
{
|
||||
metrics
|
||||
.estimated_flops_per_gpu
|
||||
.get_or_create(&labels)
|
||||
.inc_by(perf_stats.num_flops_per_gpu);
|
||||
metrics
|
||||
.estimated_read_bytes_per_gpu
|
||||
.get_or_create(&labels)
|
||||
.inc_by(perf_stats.num_read_bytes_per_gpu);
|
||||
metrics
|
||||
.estimated_write_bytes_per_gpu
|
||||
.get_or_create(&labels)
|
||||
.inc_by(perf_stats.num_write_bytes_per_gpu);
|
||||
}
|
||||
|
||||
// Sampled KV-cache residency histograms.
|
||||
if !stats.kv_cache_eviction_events.is_empty() {
|
||||
let kv_block_lifetime_seconds = metrics.kv_block_lifetime_seconds.get_or_create(&labels);
|
||||
let kv_block_idle_before_evict_seconds =
|
||||
metrics.kv_block_idle_before_evict_seconds.get_or_create(&labels);
|
||||
let kv_block_reuse_gap_seconds = metrics.kv_block_reuse_gap_seconds.get_or_create(&labels);
|
||||
|
||||
for event in &stats.kv_cache_eviction_events {
|
||||
kv_block_lifetime_seconds.observe(event.lifetime_seconds);
|
||||
kv_block_idle_before_evict_seconds.observe(event.idle_seconds);
|
||||
for reuse_gap_seconds in &event.reuse_gaps_seconds {
|
||||
kv_block_reuse_gap_seconds.observe(*reuse_gap_seconds);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,251 @@
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use enum_as_inner::EnumAsInner;
|
||||
|
||||
use super::{EngineCoreOutput, EngineCoreOutputs, UtilityOutput};
|
||||
use crate::protocol::stats::SchedulerStats;
|
||||
|
||||
/// Data-parallel control notifications multiplexed through `EngineCoreOutputs`.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum DpControlMessage {
|
||||
WaveComplete(u32),
|
||||
StartWave(u32),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct RequestBatchOutputs {
|
||||
pub engine_index: u32,
|
||||
pub outputs: Vec<EngineCoreOutput>,
|
||||
pub scheduler_stats: Option<Box<SchedulerStats>>,
|
||||
pub timestamp: f64,
|
||||
pub finished_requests: Option<BTreeSet<String>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct UtilityCallOutput {
|
||||
pub engine_index: u32,
|
||||
pub timestamp: f64,
|
||||
pub output: UtilityOutput,
|
||||
}
|
||||
|
||||
/// Semantic classification of a raw `EngineCoreOutputs` message.
|
||||
///
|
||||
/// Python currently uses one product-shaped wire struct for several distinct
|
||||
/// output families. This enum exposes those families more explicitly without
|
||||
/// changing the wire format.
|
||||
#[derive(Debug, Clone, PartialEq, EnumAsInner)]
|
||||
pub enum ClassifiedEngineCoreOutputs {
|
||||
RequestBatch(RequestBatchOutputs),
|
||||
Utility(UtilityCallOutput),
|
||||
DpControl {
|
||||
engine_index: u32,
|
||||
timestamp: f64,
|
||||
control: DpControlMessage,
|
||||
},
|
||||
/// Fallback for wire-shape combinations that do not map cleanly onto the
|
||||
/// current semantic families.
|
||||
Other(EngineCoreOutputs),
|
||||
}
|
||||
|
||||
impl EngineCoreOutputs {
|
||||
/// Classify the raw wire message into a more semantic Rust enum.
|
||||
pub fn classify(self) -> ClassifiedEngineCoreOutputs {
|
||||
let has_request_payload = !self.outputs.is_empty()
|
||||
|| self.scheduler_stats.is_some()
|
||||
|| self.finished_requests.is_some();
|
||||
|
||||
match (
|
||||
has_request_payload,
|
||||
&self.utility_output,
|
||||
&self.wave_complete,
|
||||
&self.start_wave,
|
||||
) {
|
||||
(true, None, None, None) => {
|
||||
ClassifiedEngineCoreOutputs::RequestBatch(RequestBatchOutputs {
|
||||
engine_index: self.engine_index,
|
||||
outputs: self.outputs,
|
||||
scheduler_stats: self.scheduler_stats,
|
||||
timestamp: self.timestamp,
|
||||
finished_requests: self.finished_requests,
|
||||
})
|
||||
}
|
||||
(false, Some(_), None, None) => {
|
||||
ClassifiedEngineCoreOutputs::Utility(UtilityCallOutput {
|
||||
engine_index: self.engine_index,
|
||||
timestamp: self.timestamp,
|
||||
output: self.utility_output.unwrap(),
|
||||
})
|
||||
}
|
||||
(false, None, Some(_), None) => ClassifiedEngineCoreOutputs::DpControl {
|
||||
engine_index: self.engine_index,
|
||||
timestamp: self.timestamp,
|
||||
control: DpControlMessage::WaveComplete(self.wave_complete.unwrap()),
|
||||
},
|
||||
(false, None, None, Some(_)) => ClassifiedEngineCoreOutputs::DpControl {
|
||||
engine_index: self.engine_index,
|
||||
timestamp: self.timestamp,
|
||||
control: DpControlMessage::StartWave(self.start_wave.unwrap()),
|
||||
},
|
||||
_ => ClassifiedEngineCoreOutputs::Other(self),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use super::*;
|
||||
use crate::protocol::EngineCoreOutput;
|
||||
|
||||
#[test]
|
||||
fn engine_core_outputs_classify_request_batch() {
|
||||
let outputs = EngineCoreOutputs {
|
||||
outputs: vec![EngineCoreOutput {
|
||||
request_id: "req-1".to_string(),
|
||||
new_token_ids: vec![7],
|
||||
..Default::default()
|
||||
}],
|
||||
finished_requests: Some(BTreeSet::from(["req-1".to_string()])),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
expect_test::expect![[r#"
|
||||
RequestBatch(
|
||||
RequestBatchOutputs {
|
||||
engine_index: 0,
|
||||
outputs: [
|
||||
EngineCoreOutput {
|
||||
request_id: "req-1",
|
||||
new_token_ids: [
|
||||
7,
|
||||
],
|
||||
new_logprobs: None,
|
||||
new_prompt_logprobs_tensors: None,
|
||||
pooling_output: None,
|
||||
finish_reason: None,
|
||||
stop_reason: None,
|
||||
events: None,
|
||||
kv_transfer_params: None,
|
||||
trace_headers: None,
|
||||
prefill_stats: None,
|
||||
routed_experts: None,
|
||||
num_nans_in_logits: 0,
|
||||
},
|
||||
],
|
||||
scheduler_stats: None,
|
||||
timestamp: 0.0,
|
||||
finished_requests: Some(
|
||||
{
|
||||
"req-1",
|
||||
},
|
||||
),
|
||||
},
|
||||
)
|
||||
"#]]
|
||||
.assert_debug_eq(&outputs.classify());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn engine_core_outputs_classify_utility() {
|
||||
let outputs = EngineCoreOutputs {
|
||||
utility_output: Some(UtilityOutput {
|
||||
call_id: 42,
|
||||
failure_message: None,
|
||||
result: None,
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
expect_test::expect![[r#"
|
||||
Utility(
|
||||
UtilityCallOutput {
|
||||
engine_index: 0,
|
||||
timestamp: 0.0,
|
||||
output: UtilityOutput {
|
||||
call_id: 42,
|
||||
failure_message: None,
|
||||
result: None,
|
||||
},
|
||||
},
|
||||
)
|
||||
"#]]
|
||||
.assert_debug_eq(&outputs.classify());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn engine_core_outputs_classify_control() {
|
||||
let outputs = EngineCoreOutputs {
|
||||
start_wave: Some(3),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
expect_test::expect![[r#"
|
||||
DpControl {
|
||||
engine_index: 0,
|
||||
timestamp: 0.0,
|
||||
control: StartWave(
|
||||
3,
|
||||
),
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(&outputs.classify());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn engine_core_outputs_classify_mixed_shape_as_raw() {
|
||||
let outputs = EngineCoreOutputs {
|
||||
outputs: vec![EngineCoreOutput {
|
||||
request_id: "req-1".to_string(),
|
||||
new_token_ids: vec![7],
|
||||
..Default::default()
|
||||
}],
|
||||
utility_output: Some(UtilityOutput {
|
||||
call_id: 1,
|
||||
failure_message: None,
|
||||
result: None,
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
expect_test::expect![[r#"
|
||||
Other(
|
||||
EngineCoreOutputs {
|
||||
engine_index: 0,
|
||||
outputs: [
|
||||
EngineCoreOutput {
|
||||
request_id: "req-1",
|
||||
new_token_ids: [
|
||||
7,
|
||||
],
|
||||
new_logprobs: None,
|
||||
new_prompt_logprobs_tensors: None,
|
||||
pooling_output: None,
|
||||
finish_reason: None,
|
||||
stop_reason: None,
|
||||
events: None,
|
||||
kv_transfer_params: None,
|
||||
trace_headers: None,
|
||||
prefill_stats: None,
|
||||
routed_experts: None,
|
||||
num_nans_in_logits: 0,
|
||||
},
|
||||
],
|
||||
scheduler_stats: None,
|
||||
timestamp: 0.0,
|
||||
utility_output: Some(
|
||||
UtilityOutput {
|
||||
call_id: 1,
|
||||
failure_message: None,
|
||||
result: None,
|
||||
},
|
||||
),
|
||||
finished_requests: None,
|
||||
wave_complete: None,
|
||||
start_wave: None,
|
||||
},
|
||||
)
|
||||
"#]]
|
||||
.assert_debug_eq(&outputs.classify());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Effective model dtype reported by the engine after config resolution.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum ModelDtype {
|
||||
#[serde(rename = "float16")]
|
||||
Float16,
|
||||
#[serde(rename = "bfloat16")]
|
||||
BFloat16,
|
||||
#[serde(rename = "float32")]
|
||||
Float32,
|
||||
}
|
||||
|
||||
impl ModelDtype {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Float16 => "float16",
|
||||
Self::BFloat16 => "bfloat16",
|
||||
Self::Float32 => "float32",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::ModelDtype;
|
||||
|
||||
#[test]
|
||||
fn serde_uses_protocol_dtype_strings() {
|
||||
assert_eq!(
|
||||
serde_json::to_value(ModelDtype::Float16).unwrap(),
|
||||
serde_json::json!("float16")
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::from_value::<ModelDtype>(serde_json::json!("bfloat16")).unwrap(),
|
||||
ModelDtype::BFloat16
|
||||
);
|
||||
assert_eq!(ModelDtype::Float32.as_str(), "float32");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::protocol::{ModelDtype, OpaqueValue};
|
||||
|
||||
/// Decoded engine startup-handshake payload sent on the handshake socket.
|
||||
///
|
||||
/// Original Python payload construction:
|
||||
/// <https://github.com/vllm-project/vllm/blob/c8d98f81f6/vllm/v1/engine/core.py#L1000-L1035>
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct ReadyMessage {
|
||||
#[serde(default)]
|
||||
pub status: Option<String>,
|
||||
#[serde(default)]
|
||||
pub local: Option<bool>,
|
||||
#[serde(default)]
|
||||
pub headless: Option<bool>,
|
||||
#[serde(default)]
|
||||
pub parallel_config_hash: Option<String>,
|
||||
}
|
||||
|
||||
/// Post-initialization configuration sent from each engine on the input socket
|
||||
/// registration message, after the handshake completes.
|
||||
///
|
||||
/// Contains values that may differ from the original config (e.g.
|
||||
/// `max_model_len` after KV cache auto-fitting, `num_gpu_blocks` after
|
||||
/// profiling).
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/c8d98f81f6/vllm/v1/engine/__init__.py#L67-L77>
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct EngineCoreReadyResponse {
|
||||
/// Engine-reported maximum model context length (auto-fitted after
|
||||
/// KV cache profiling and may differ from the original config value).
|
||||
pub max_model_len: u64,
|
||||
/// Number of GPU blocks available for KV cache on this engine.
|
||||
pub num_gpu_blocks: u64,
|
||||
/// DP coordinator stats publish address, if applicable.
|
||||
pub dp_stats_address: Option<String>,
|
||||
/// Effective model dtype after Python vLLM resolves `--dtype`.
|
||||
// TODO: This is currently not wired up on the engine side. After it's added, remove `Option`
|
||||
// and `serde(default)`.
|
||||
#[serde(default)]
|
||||
pub dtype: Option<ModelDtype>,
|
||||
}
|
||||
|
||||
/// Frontend-owned ZMQ addresses that are sent to the engine during startup
|
||||
/// handshake initialization.
|
||||
///
|
||||
/// Original Python definition (`EngineZmqAddresses`):
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/utils.py#L53-L67>
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct HandshakeAddresses {
|
||||
pub inputs: Vec<String>,
|
||||
pub outputs: Vec<String>,
|
||||
pub coordinator_input: Option<String>,
|
||||
pub coordinator_output: Option<String>,
|
||||
pub frontend_stats_publish_address: Option<String>,
|
||||
}
|
||||
|
||||
/// Startup handshake payload sent from the frontend to initialize an engine
|
||||
/// after receiving `HELLO`.
|
||||
///
|
||||
/// Original Python definition (`EngineHandshakeMetadata`):
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/utils.py#L69-L77>
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct HandshakeInitMessage {
|
||||
pub addresses: HandshakeAddresses,
|
||||
pub parallel_config: BTreeMap<String, OpaqueValue>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::EngineCoreReadyResponse;
|
||||
use crate::protocol::ModelDtype;
|
||||
|
||||
#[test]
|
||||
fn ready_response_accepts_effective_dtype() {
|
||||
let response: EngineCoreReadyResponse = serde_json::from_value(serde_json::json!({
|
||||
"max_model_len": 4096,
|
||||
"num_gpu_blocks": 2,
|
||||
"dp_stats_address": null,
|
||||
"dtype": "bfloat16"
|
||||
}))
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(response.dtype, Some(ModelDtype::BFloat16));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,330 @@
|
||||
mod array;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
mod wire;
|
||||
|
||||
use std::ops::{Deref, DerefMut};
|
||||
|
||||
use enum_as_inner::EnumAsInner;
|
||||
use serde::{Deserialize, Deserializer, Serialize};
|
||||
|
||||
use self::wire::*;
|
||||
use super::{EngineCoreOutput, EngineCoreOutputs, decode_msgpack};
|
||||
use crate::error::{Error, Result, bail_ext_value_decode, ext_value_decode};
|
||||
use crate::protocol::tensor::{WireArrayData, WireNdArray};
|
||||
|
||||
/// One token candidate and its logprob metadata for a single sequence position.
|
||||
///
|
||||
/// The first entry in a [`PositionLogprobs`] is always the sampled/selected
|
||||
/// token for that position. Any remaining entries follow the engine's returned
|
||||
/// top-k candidate order.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct TokenLogprob {
|
||||
pub token_id: u32,
|
||||
pub logprob: f32,
|
||||
/// The sampled/selected token uses its actual vocab rank. Remaining entries
|
||||
/// use 1-based top-k ranks matching the engine's returned candidate
|
||||
/// order.
|
||||
pub rank: u32,
|
||||
}
|
||||
|
||||
/// Logprob payload for one sequence position.
|
||||
///
|
||||
/// This is the semantic Rust representation used by the public client API after
|
||||
/// the lower-level ndarray/tensor wire payload has been decoded.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct PositionLogprobs {
|
||||
pub entries: Vec<TokenLogprob>,
|
||||
}
|
||||
|
||||
impl PositionLogprobs {
|
||||
/// Convert one decoded logprobs row into this per-position form by grouping
|
||||
/// each token/logprob pair together with the sampled/selected token's
|
||||
/// actual vocab rank.
|
||||
fn from_decoded_row(token_ids: &[u32], logprobs: &[f32], sampled_rank: u32) -> Result<Self> {
|
||||
if token_ids.len() != logprobs.len() {
|
||||
bail_ext_value_decode!(
|
||||
"logprobs row length mismatch: token_ids={}, logprobs={}",
|
||||
token_ids.len(),
|
||||
logprobs.len()
|
||||
);
|
||||
}
|
||||
if sampled_rank == 0 {
|
||||
bail_ext_value_decode!("token_ranks must be >= 1 for decoded engine-core logprobs");
|
||||
}
|
||||
|
||||
let mut entries = Vec::with_capacity(token_ids.len());
|
||||
for (index, (&token_id, &logprob)) in token_ids.iter().zip(logprobs.iter()).enumerate() {
|
||||
let rank = if index == 0 {
|
||||
sampled_rank
|
||||
} else {
|
||||
index as u32
|
||||
};
|
||||
entries.push(TokenLogprob {
|
||||
token_id,
|
||||
logprob,
|
||||
rank,
|
||||
});
|
||||
}
|
||||
Ok(Self { entries })
|
||||
}
|
||||
}
|
||||
|
||||
/// Decoded per-request logprobs payload for one engine-core output.
|
||||
///
|
||||
/// Unlike the Python wire payload, this public Rust type is already fully
|
||||
/// semantic: one [`PositionLogprobs`] per scored position, each containing the
|
||||
/// sampled/selected token plus any returned top-k alternatives for that same
|
||||
/// position.
|
||||
///
|
||||
/// The Python engine still sends logprobs as ndarray/tensor-shaped wire tuples.
|
||||
/// Rust resolves that lower-level representation during decode and exposes only
|
||||
/// this per-position form to callers.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct Logprobs {
|
||||
/// One decoded logprobs record per scored position in this engine-core
|
||||
/// output.
|
||||
pub positions: Vec<PositionLogprobs>,
|
||||
}
|
||||
|
||||
impl Logprobs {
|
||||
/// Returns the number of scored positions in this payload.
|
||||
pub fn len(&self) -> usize {
|
||||
self.positions.len()
|
||||
}
|
||||
|
||||
/// Returns whether the payload contains no scored positions.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.positions.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
/// Output field wrapper that is initially deserialized from the Python wire
|
||||
/// shape, then resolved into [`Logprobs`] before the decoded message is
|
||||
/// returned to callers.
|
||||
#[derive(Clone, PartialEq, Debug, EnumAsInner)]
|
||||
pub enum MaybeWireLogprobs {
|
||||
/// The logprobs are still in the wire format and need to be resolved by
|
||||
/// looking up aux frames and decoding raw views. Should only be used
|
||||
/// internally during deserialization.
|
||||
Wire(Box<WireLogprobs>),
|
||||
/// The actual decoded logprobs value,
|
||||
Direct(Logprobs),
|
||||
}
|
||||
|
||||
impl Deref for MaybeWireLogprobs {
|
||||
type Target = Logprobs;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
match self {
|
||||
Self::Wire(_) => panic!("Logprobs is still in wire format"),
|
||||
Self::Direct(value) => value,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl DerefMut for MaybeWireLogprobs {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
match self {
|
||||
Self::Wire(_) => panic!("Logprobs is still in wire format"),
|
||||
Self::Direct(value) => value,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for MaybeWireLogprobs {
|
||||
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
// When deserializing, it's always in the wire form.
|
||||
WireLogprobs::deserialize(deserializer).map(|v| Self::Wire(Box::new(v)))
|
||||
}
|
||||
}
|
||||
|
||||
impl Serialize for MaybeWireLogprobs {
|
||||
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
// For testing purposes only. We don't actually serialize it into aux frames.
|
||||
match self {
|
||||
Self::Wire(value) => value.serialize(serializer),
|
||||
Self::Direct(value) => WireLogprobs::from_direct(value)
|
||||
.map_err(serde::ser::Error::custom)?
|
||||
.serialize(serializer),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl MaybeWireLogprobs {
|
||||
/// Resolve the wire representation into decoded logprobs by looking up aux
|
||||
/// frames and decoding raw views as needed.
|
||||
fn resolve<Frame>(self, frames: &[Frame], field_prefix: &str) -> Result<Self>
|
||||
where
|
||||
Frame: AsRef<[u8]>,
|
||||
{
|
||||
match self {
|
||||
Self::Direct(value) => Ok(Self::Direct(value)),
|
||||
Self::Wire(value) => value.resolve(frames, field_prefix).map(Self::Direct),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl EngineCoreOutputs {
|
||||
/// Resolve all wire-format fields in-place by looking up aux frames and
|
||||
/// decoding raw-view payloads as needed.
|
||||
fn resolve_in_place<Frame>(&mut self, frames: &[Frame]) -> Result<()>
|
||||
where
|
||||
Frame: AsRef<[u8]>,
|
||||
{
|
||||
for output in &mut self.outputs {
|
||||
output.resolve_in_place(frames)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl EngineCoreOutput {
|
||||
/// Resolve all wire-format fields in-place by looking up aux frames and
|
||||
/// decoding raw-view payloads as needed.
|
||||
fn resolve_in_place<Frame>(&mut self, frames: &[Frame]) -> Result<()>
|
||||
where
|
||||
Frame: AsRef<[u8]>,
|
||||
{
|
||||
self.new_logprobs = (self.new_logprobs.take())
|
||||
.map(|value| value.resolve(frames, "new_logprobs"))
|
||||
.transpose()?;
|
||||
self.new_prompt_logprobs_tensors = (self.new_prompt_logprobs_tensors.take())
|
||||
.map(|value| value.resolve(frames, "new_prompt_logprobs_tensors"))
|
||||
.transpose()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl WireLogprobs {
|
||||
/// Convert semantic per-position logprobs into the Python wire tuple shape.
|
||||
///
|
||||
/// This exists mainly so Rust-side tests can inject semantic logprobs into
|
||||
/// mocked engine-core outputs without manually building ndarray
|
||||
/// raw-view tuples.
|
||||
fn from_direct(value: &Logprobs) -> std::result::Result<Self, String> {
|
||||
let rows = value.positions.len();
|
||||
let cols = value.positions.first().map(|position| position.entries.len()).unwrap_or(0);
|
||||
|
||||
let mut token_ids = Vec::with_capacity(rows.saturating_mul(cols).saturating_mul(8));
|
||||
let mut logprobs = Vec::with_capacity(rows.saturating_mul(cols).saturating_mul(4));
|
||||
let mut token_ranks = Vec::with_capacity(rows.saturating_mul(8));
|
||||
|
||||
for (row_index, position) in value.positions.iter().enumerate() {
|
||||
if position.entries.len() != cols {
|
||||
return Err(format!(
|
||||
"logprobs row {row_index} length mismatch: expected {cols}, got {}",
|
||||
position.entries.len()
|
||||
));
|
||||
}
|
||||
let Some((sampled, _)) = position.entries.split_first() else {
|
||||
return Err(format!("logprobs row {row_index} is empty"));
|
||||
};
|
||||
|
||||
token_ranks.extend_from_slice(&(sampled.rank as i64).to_le_bytes());
|
||||
for entry in &position.entries {
|
||||
token_ids.extend_from_slice(&(entry.token_id as i64).to_le_bytes());
|
||||
logprobs.extend_from_slice(&entry.logprob.to_le_bytes());
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
logprob_token_ids: WireNdArray {
|
||||
dtype: "<i8".to_string(),
|
||||
shape: vec![rows, cols],
|
||||
data: WireArrayData::RawView(token_ids),
|
||||
},
|
||||
logprobs: WireNdArray {
|
||||
dtype: "<f4".to_string(),
|
||||
shape: vec![rows, cols],
|
||||
data: WireArrayData::RawView(logprobs),
|
||||
},
|
||||
token_ranks: WireNdArray {
|
||||
dtype: "<i8".to_string(),
|
||||
shape: vec![rows],
|
||||
data: WireArrayData::RawView(token_ranks),
|
||||
},
|
||||
cu_num_generated_tokens: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Resolve the wire-format logprobs into semantic [`Logprobs`] records by
|
||||
/// looking up aux frames, decoding raw views, and grouping each row
|
||||
/// into one [`PositionLogprobs`].
|
||||
fn resolve<Frame>(self, frames: &[Frame], field_prefix: &str) -> Result<Logprobs>
|
||||
where
|
||||
Frame: AsRef<[u8]>,
|
||||
{
|
||||
if let Some(indices) = self.cu_num_generated_tokens {
|
||||
bail_ext_value_decode!(
|
||||
"{field_prefix}.cu_num_generated_tokens: \
|
||||
expected None for per-request engine-core logprobs payload, got {indices:?}"
|
||||
);
|
||||
}
|
||||
|
||||
let token_ids = array::decode_array2_u32(
|
||||
self.logprob_token_ids,
|
||||
&format!("{field_prefix}.logprob_token_ids"),
|
||||
frames,
|
||||
)?;
|
||||
let logprobs =
|
||||
array::decode_array2_f32(self.logprobs, &format!("{field_prefix}.logprobs"), frames)?;
|
||||
let token_ranks = array::decode_array1_u32(
|
||||
self.token_ranks,
|
||||
&format!("{field_prefix}.token_ranks"),
|
||||
frames,
|
||||
)?;
|
||||
|
||||
if token_ids.rows != logprobs.rows || token_ids.cols != logprobs.cols {
|
||||
bail_ext_value_decode!(
|
||||
"{field_prefix}: row shape mismatch between token ids ({}, {}) and logprobs ({}, {})",
|
||||
token_ids.rows,
|
||||
token_ids.cols,
|
||||
logprobs.rows,
|
||||
logprobs.cols
|
||||
);
|
||||
}
|
||||
if token_ids.rows != token_ranks.len() {
|
||||
bail_ext_value_decode!(
|
||||
"{field_prefix}: token_ranks length {} does not match row count {}",
|
||||
token_ranks.len(),
|
||||
token_ids.rows
|
||||
);
|
||||
}
|
||||
|
||||
let mut positions = Vec::with_capacity(token_ids.rows);
|
||||
for ((token_ids_row, logprobs_row), sampled_rank) in token_ids
|
||||
.data
|
||||
.chunks(token_ids.cols)
|
||||
.zip(logprobs.data.chunks(logprobs.cols))
|
||||
.zip(token_ranks)
|
||||
{
|
||||
positions.push(PositionLogprobs::from_decoded_row(
|
||||
token_ids_row,
|
||||
logprobs_row,
|
||||
sampled_rank,
|
||||
)?);
|
||||
}
|
||||
|
||||
Ok(Logprobs { positions })
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode one ordinary or multipart engine-core output message into the strong
|
||||
/// typed public protocol shape.
|
||||
pub fn decode_engine_core_outputs<Frame>(frames: &[Frame]) -> Result<EngineCoreOutputs>
|
||||
where
|
||||
Frame: AsRef<[u8]>,
|
||||
{
|
||||
let first_frame = frames.first().ok_or_else(|| ext_value_decode!("missing output frame"))?;
|
||||
|
||||
let mut outputs: EngineCoreOutputs = decode_msgpack(first_frame.as_ref())?;
|
||||
outputs.resolve_in_place(frames)?;
|
||||
Ok(outputs)
|
||||
}
|
||||
@@ -0,0 +1,306 @@
|
||||
use std::io::Cursor;
|
||||
|
||||
use byteorder::{BigEndian, LittleEndian, NativeEndian, ReadBytesExt};
|
||||
use itertools::Itertools as _;
|
||||
|
||||
use crate::error::{Error, Result, ext_value_decode};
|
||||
use crate::protocol::tensor::{ShapeExt as _, WireArrayData, WireNdArray};
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum ScalarType {
|
||||
I32,
|
||||
I64,
|
||||
F32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum Endianness {
|
||||
Little,
|
||||
Big,
|
||||
Native,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub(super) struct DecodedArray2<T> {
|
||||
pub rows: usize,
|
||||
pub cols: usize,
|
||||
pub data: Vec<T>,
|
||||
}
|
||||
|
||||
pub(super) fn decode_array2_u32<Frame>(
|
||||
value: WireNdArray,
|
||||
field: &str,
|
||||
frames: &[Frame],
|
||||
) -> Result<DecodedArray2<u32>>
|
||||
where
|
||||
Frame: AsRef<[u8]>,
|
||||
{
|
||||
let (shape, bytes, scalar, endianness) =
|
||||
decode_array_metadata(value, field, frames, &[ScalarType::I32, ScalarType::I64])?;
|
||||
if shape.len() != 2 {
|
||||
return Err(decode_error(
|
||||
field,
|
||||
&format!("expected rank-2 array, got rank {}", shape.len()),
|
||||
));
|
||||
}
|
||||
|
||||
let data = match scalar {
|
||||
ScalarType::I32 => decode_i32_vec(&bytes, endianness, field)?
|
||||
.into_iter()
|
||||
.map(|value| convert_to_u32(value, field))
|
||||
.try_collect()?,
|
||||
ScalarType::I64 => decode_i64_vec(&bytes, endianness, field)?
|
||||
.into_iter()
|
||||
.map(|value| convert_to_u32(value, field))
|
||||
.try_collect()?,
|
||||
ScalarType::F32 => unreachable!("scalar validation should reject f32"),
|
||||
};
|
||||
Ok(DecodedArray2 {
|
||||
rows: shape[0],
|
||||
cols: shape[1],
|
||||
data,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn decode_array1_u32<Frame>(
|
||||
value: WireNdArray,
|
||||
field: &str,
|
||||
frames: &[Frame],
|
||||
) -> Result<Vec<u32>>
|
||||
where
|
||||
Frame: AsRef<[u8]>,
|
||||
{
|
||||
let (shape, bytes, scalar, endianness) =
|
||||
decode_array_metadata(value, field, frames, &[ScalarType::I32, ScalarType::I64])?;
|
||||
if shape.len() != 1 {
|
||||
return Err(decode_error(
|
||||
field,
|
||||
&format!("expected rank-1 array, got rank {}", shape.len()),
|
||||
));
|
||||
}
|
||||
|
||||
let data = match scalar {
|
||||
ScalarType::I32 => decode_i32_vec(&bytes, endianness, field)?
|
||||
.into_iter()
|
||||
.map(|value| convert_to_u32(value, field))
|
||||
.try_collect()?,
|
||||
ScalarType::I64 => decode_i64_vec(&bytes, endianness, field)?
|
||||
.into_iter()
|
||||
.map(|value| convert_to_u32(value, field))
|
||||
.try_collect()?,
|
||||
ScalarType::F32 => unreachable!("scalar validation should reject f32"),
|
||||
};
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
pub(super) fn decode_array2_f32<Frame>(
|
||||
value: WireNdArray,
|
||||
field: &str,
|
||||
frames: &[Frame],
|
||||
) -> Result<DecodedArray2<f32>>
|
||||
where
|
||||
Frame: AsRef<[u8]>,
|
||||
{
|
||||
let (shape, bytes, _, endianness) =
|
||||
decode_array_metadata(value, field, frames, &[ScalarType::F32])?;
|
||||
if shape.len() != 2 {
|
||||
return Err(decode_error(
|
||||
field,
|
||||
&format!("expected rank-2 array, got rank {}", shape.len()),
|
||||
));
|
||||
}
|
||||
|
||||
let data = decode_f32_vec(&bytes, endianness, field)?;
|
||||
Ok(DecodedArray2 {
|
||||
rows: shape[0],
|
||||
cols: shape[1],
|
||||
data,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn decode_array_metadata<Frame>(
|
||||
value: WireNdArray,
|
||||
field: &str,
|
||||
frames: &[Frame],
|
||||
expected_scalars: &[ScalarType],
|
||||
) -> Result<(Vec<usize>, Vec<u8>, ScalarType, Endianness)>
|
||||
where
|
||||
Frame: AsRef<[u8]>,
|
||||
{
|
||||
let WireNdArray { dtype, shape, data } = value;
|
||||
let (scalar, endianness) = parse_dtype(&dtype, field)?;
|
||||
if !expected_scalars.contains(&scalar) {
|
||||
return Err(decode_error(
|
||||
field,
|
||||
&format!("expected dtype in {:?}, got {}", expected_scalars, dtype),
|
||||
));
|
||||
}
|
||||
|
||||
let bytes = resolve_array_bytes(data, field, frames)?;
|
||||
validate_byte_length(shape.as_slice(), bytes.len(), field, scalar)?;
|
||||
Ok((shape, bytes, scalar, endianness))
|
||||
}
|
||||
|
||||
pub(super) fn parse_dtype(dtype: &str, field: &str) -> Result<(ScalarType, Endianness)> {
|
||||
let (endianness, body) = match dtype.as_bytes().first().copied() {
|
||||
Some(b'<') => (Endianness::Little, &dtype[1..]),
|
||||
Some(b'>') => (Endianness::Big, &dtype[1..]),
|
||||
Some(b'=') => (Endianness::Native, &dtype[1..]),
|
||||
Some(b'|') => (Endianness::Native, &dtype[1..]),
|
||||
_ => (Endianness::Native, dtype),
|
||||
};
|
||||
|
||||
let scalar = match body {
|
||||
"i4" | "int32" => ScalarType::I32,
|
||||
"i8" | "int64" => ScalarType::I64,
|
||||
"f4" | "float32" => ScalarType::F32,
|
||||
_ => {
|
||||
return Err(decode_error(
|
||||
field,
|
||||
&format!("unsupported dtype string {dtype:?}"),
|
||||
));
|
||||
}
|
||||
};
|
||||
Ok((scalar, endianness))
|
||||
}
|
||||
|
||||
pub(super) fn resolve_array_bytes<Frame>(
|
||||
value: WireArrayData,
|
||||
field: &str,
|
||||
frames: &[Frame],
|
||||
) -> Result<Vec<u8>>
|
||||
where
|
||||
Frame: AsRef<[u8]>,
|
||||
{
|
||||
match value {
|
||||
WireArrayData::RawView(bytes) => Ok(bytes),
|
||||
WireArrayData::AuxIndex(index) => {
|
||||
let frame = frames.get(index).ok_or_else(|| {
|
||||
decode_error(
|
||||
field,
|
||||
&format!(
|
||||
"aux frame index {index} out of range for {} frames",
|
||||
frames.len()
|
||||
),
|
||||
)
|
||||
})?;
|
||||
Ok(frame.as_ref().to_vec())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn validate_byte_length(
|
||||
shape: &[usize],
|
||||
byte_len: usize,
|
||||
field: &str,
|
||||
scalar: ScalarType,
|
||||
) -> Result<()> {
|
||||
let element_count = shape
|
||||
.checked_numel()
|
||||
.ok_or_else(|| decode_error(field, "shape element count overflowed usize"))?;
|
||||
let element_size = match scalar {
|
||||
ScalarType::I32 | ScalarType::F32 => 4,
|
||||
ScalarType::I64 => 8,
|
||||
};
|
||||
let expected = element_count
|
||||
.checked_mul(element_size)
|
||||
.ok_or_else(|| decode_error(field, "byte length overflowed usize"))?;
|
||||
if expected != byte_len {
|
||||
return Err(decode_error(
|
||||
field,
|
||||
&format!("byte length mismatch: expected {expected}, got {byte_len}"),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn decode_i32_vec(
|
||||
bytes: &[u8],
|
||||
endianness: Endianness,
|
||||
field: &str,
|
||||
) -> Result<Vec<i32>> {
|
||||
if !bytes.len().is_multiple_of(4) {
|
||||
return Err(decode_error(
|
||||
field,
|
||||
&format!("byte length {} is not divisible by 4", bytes.len()),
|
||||
));
|
||||
}
|
||||
let mut cursor = Cursor::new(bytes);
|
||||
let mut values = Vec::with_capacity(bytes.len() / 4);
|
||||
while (cursor.position() as usize) < bytes.len() {
|
||||
let value = match endianness {
|
||||
Endianness::Little => cursor.read_i32::<LittleEndian>(),
|
||||
Endianness::Big => cursor.read_i32::<BigEndian>(),
|
||||
Endianness::Native => cursor.read_i32::<NativeEndian>(),
|
||||
}
|
||||
.map_err(|error| decode_error(field, &format!("failed to read i32 payload: {error}")))?;
|
||||
values.push(value);
|
||||
}
|
||||
Ok(values)
|
||||
}
|
||||
|
||||
pub(super) fn decode_f32_vec(
|
||||
bytes: &[u8],
|
||||
endianness: Endianness,
|
||||
field: &str,
|
||||
) -> Result<Vec<f32>> {
|
||||
if !bytes.len().is_multiple_of(4) {
|
||||
return Err(decode_error(
|
||||
field,
|
||||
&format!("byte length {} is not divisible by 4", bytes.len()),
|
||||
));
|
||||
}
|
||||
let mut cursor = Cursor::new(bytes);
|
||||
let mut values = Vec::with_capacity(bytes.len() / 4);
|
||||
while (cursor.position() as usize) < bytes.len() {
|
||||
let value = match endianness {
|
||||
Endianness::Little => cursor.read_f32::<LittleEndian>(),
|
||||
Endianness::Big => cursor.read_f32::<BigEndian>(),
|
||||
Endianness::Native => cursor.read_f32::<NativeEndian>(),
|
||||
}
|
||||
.map_err(|error| decode_error(field, &format!("failed to read f32 payload: {error}")))?;
|
||||
values.push(value);
|
||||
}
|
||||
Ok(values)
|
||||
}
|
||||
|
||||
pub(super) fn decode_i64_vec(
|
||||
bytes: &[u8],
|
||||
endianness: Endianness,
|
||||
field: &str,
|
||||
) -> Result<Vec<i64>> {
|
||||
if !bytes.len().is_multiple_of(8) {
|
||||
return Err(decode_error(
|
||||
field,
|
||||
&format!("byte length {} is not divisible by 8", bytes.len()),
|
||||
));
|
||||
}
|
||||
let mut cursor = Cursor::new(bytes);
|
||||
let mut values = Vec::with_capacity(bytes.len() / 8);
|
||||
while (cursor.position() as usize) < bytes.len() {
|
||||
let value = match endianness {
|
||||
Endianness::Little => cursor.read_i64::<LittleEndian>(),
|
||||
Endianness::Big => cursor.read_i64::<BigEndian>(),
|
||||
Endianness::Native => cursor.read_i64::<NativeEndian>(),
|
||||
}
|
||||
.map_err(|error| decode_error(field, &format!("failed to read i64 payload: {error}")))?;
|
||||
values.push(value);
|
||||
}
|
||||
Ok(values)
|
||||
}
|
||||
|
||||
fn convert_to_u32<I>(value: I, field: &str) -> Result<u32>
|
||||
where
|
||||
I: TryInto<u32> + std::fmt::Display + Copy,
|
||||
{
|
||||
value.try_into().map_err(|_| {
|
||||
decode_error(
|
||||
field,
|
||||
&format!("expected non-negative token id/rank that fits in u32, got {value}"),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn decode_error(field: &str, reason: &str) -> Error {
|
||||
ext_value_decode!("{field}: {reason}")
|
||||
}
|
||||
@@ -0,0 +1,302 @@
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use bytes::Bytes;
|
||||
use rmpv::Value;
|
||||
|
||||
use super::{Logprobs, PositionLogprobs, TokenLogprob, decode_engine_core_outputs};
|
||||
use crate::protocol::EngineCoreFinishReason;
|
||||
|
||||
fn encode_value(value: &Value) -> Vec<u8> {
|
||||
let mut out = Vec::new();
|
||||
rmpv::encode::write_value(&mut out, value).unwrap();
|
||||
out
|
||||
}
|
||||
|
||||
fn output_wire_with_custom_fields(
|
||||
new_logprobs: Option<Value>,
|
||||
prompt_logprobs: Option<Value>,
|
||||
) -> Value {
|
||||
Value::Array(vec![
|
||||
Value::from(0),
|
||||
Value::Array(vec![Value::Array(vec![
|
||||
Value::from("req-1"),
|
||||
Value::Array(vec![Value::from(7), Value::from(8)]),
|
||||
new_logprobs.unwrap_or(Value::Nil),
|
||||
prompt_logprobs.unwrap_or(Value::Nil),
|
||||
Value::Nil,
|
||||
Value::from(EngineCoreFinishReason::Length as u8),
|
||||
])]),
|
||||
Value::Nil,
|
||||
Value::from(0.0),
|
||||
Value::Nil,
|
||||
Value::Array(vec![Value::from("req-1")]),
|
||||
])
|
||||
}
|
||||
|
||||
fn ndarray_value(dtype: &str, shape: &[usize], data: Value) -> Value {
|
||||
Value::Array(vec![
|
||||
Value::from(dtype),
|
||||
Value::Array(shape.iter().copied().map(Value::from).collect()),
|
||||
data,
|
||||
])
|
||||
}
|
||||
|
||||
fn inline_logprobs_value() -> Value {
|
||||
let ids = Value::Ext(
|
||||
3,
|
||||
vec![
|
||||
1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0,
|
||||
0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0,
|
||||
],
|
||||
);
|
||||
let probs = Value::Ext(
|
||||
3,
|
||||
vec![
|
||||
0, 0, 128, 63, 0, 0, 0, 64, 0, 0, 64, 64, 0, 0, 128, 64, 0, 0, 160, 64, 0, 0, 192, 64,
|
||||
],
|
||||
);
|
||||
let ranks = Value::Ext(3, vec![1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0]);
|
||||
Value::Array(vec![
|
||||
ndarray_value("<i8", &[2, 3], ids),
|
||||
ndarray_value("<f4", &[2, 3], probs),
|
||||
ndarray_value("<i8", &[2], ranks),
|
||||
Value::Nil,
|
||||
])
|
||||
}
|
||||
|
||||
fn inline_prompt_logprobs_value() -> Value {
|
||||
let ids = Value::Ext(
|
||||
3,
|
||||
vec![
|
||||
10, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0,
|
||||
0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0,
|
||||
],
|
||||
);
|
||||
let probs = Value::Ext(
|
||||
3,
|
||||
vec![
|
||||
0, 0, 32, 65, 0, 0, 48, 65, 0, 0, 64, 65, 0, 0, 80, 65, 0, 0, 96, 65, 0, 0, 112, 65,
|
||||
],
|
||||
);
|
||||
let ranks = Value::Ext(3, vec![3, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0]);
|
||||
Value::Array(vec![
|
||||
ndarray_value("int64", &[2, 3], ids),
|
||||
ndarray_value("float32", &[2, 3], probs),
|
||||
ndarray_value("int64", &[2], ranks),
|
||||
Value::Nil,
|
||||
])
|
||||
}
|
||||
|
||||
fn expected_sample_logprobs() -> Logprobs {
|
||||
Logprobs {
|
||||
positions: vec![
|
||||
PositionLogprobs {
|
||||
entries: vec![
|
||||
TokenLogprob {
|
||||
token_id: 1,
|
||||
logprob: 1.0,
|
||||
rank: 1,
|
||||
},
|
||||
TokenLogprob {
|
||||
token_id: 2,
|
||||
logprob: 2.0,
|
||||
rank: 1,
|
||||
},
|
||||
TokenLogprob {
|
||||
token_id: 3,
|
||||
logprob: 3.0,
|
||||
rank: 2,
|
||||
},
|
||||
],
|
||||
},
|
||||
PositionLogprobs {
|
||||
entries: vec![
|
||||
TokenLogprob {
|
||||
token_id: 4,
|
||||
logprob: 4.0,
|
||||
rank: 2,
|
||||
},
|
||||
TokenLogprob {
|
||||
token_id: 5,
|
||||
logprob: 5.0,
|
||||
rank: 1,
|
||||
},
|
||||
TokenLogprob {
|
||||
token_id: 6,
|
||||
logprob: 6.0,
|
||||
rank: 2,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
fn expected_prompt_logprobs() -> Logprobs {
|
||||
Logprobs {
|
||||
positions: vec![
|
||||
PositionLogprobs {
|
||||
entries: vec![
|
||||
TokenLogprob {
|
||||
token_id: 10,
|
||||
logprob: 10.0,
|
||||
rank: 3,
|
||||
},
|
||||
TokenLogprob {
|
||||
token_id: 11,
|
||||
logprob: 11.0,
|
||||
rank: 1,
|
||||
},
|
||||
TokenLogprob {
|
||||
token_id: 12,
|
||||
logprob: 12.0,
|
||||
rank: 2,
|
||||
},
|
||||
],
|
||||
},
|
||||
PositionLogprobs {
|
||||
entries: vec![
|
||||
TokenLogprob {
|
||||
token_id: 13,
|
||||
logprob: 13.0,
|
||||
rank: 4,
|
||||
},
|
||||
TokenLogprob {
|
||||
token_id: 14,
|
||||
logprob: 14.0,
|
||||
rank: 1,
|
||||
},
|
||||
TokenLogprob {
|
||||
token_id: 15,
|
||||
logprob: 15.0,
|
||||
rank: 2,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decodes_inline_new_logprobs() {
|
||||
let frames = vec![Bytes::from(encode_value(&output_wire_with_custom_fields(
|
||||
Some(inline_logprobs_value()),
|
||||
None,
|
||||
)))];
|
||||
let decoded = decode_engine_core_outputs(&frames).unwrap();
|
||||
|
||||
let logprobs = decoded.outputs[0].new_logprobs.clone().unwrap().into_direct().unwrap();
|
||||
assert_eq!(logprobs, expected_sample_logprobs());
|
||||
assert_eq!(
|
||||
decoded.finished_requests,
|
||||
Some(BTreeSet::from(["req-1".to_string()]))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decodes_multipart_new_logprobs() {
|
||||
let frames = vec![
|
||||
Bytes::from(encode_value(&output_wire_with_custom_fields(
|
||||
Some(Value::Array(vec![
|
||||
ndarray_value("<i8", &[2, 3], Value::from(1)),
|
||||
ndarray_value("<f4", &[2, 3], Value::from(2)),
|
||||
ndarray_value("<i8", &[2], Value::from(3)),
|
||||
Value::Nil,
|
||||
])),
|
||||
None,
|
||||
))),
|
||||
Bytes::from_static(&[
|
||||
1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0,
|
||||
0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0,
|
||||
]),
|
||||
Bytes::from_static(&[
|
||||
0, 0, 128, 63, 0, 0, 0, 64, 0, 0, 64, 64, 0, 0, 128, 64, 0, 0, 160, 64, 0, 0, 192, 64,
|
||||
]),
|
||||
Bytes::from_static(&[1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0]),
|
||||
];
|
||||
let decoded = decode_engine_core_outputs(&frames).unwrap();
|
||||
|
||||
let logprobs = decoded.outputs[0].new_logprobs.clone().unwrap().into_direct().unwrap();
|
||||
assert_eq!(logprobs, expected_sample_logprobs());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decodes_inline_prompt_logprobs() {
|
||||
let frames = vec![Bytes::from(encode_value(&output_wire_with_custom_fields(
|
||||
None,
|
||||
Some(inline_prompt_logprobs_value()),
|
||||
)))];
|
||||
let decoded = decode_engine_core_outputs(&frames).unwrap();
|
||||
|
||||
let logprobs = decoded.outputs[0]
|
||||
.new_prompt_logprobs_tensors
|
||||
.clone()
|
||||
.unwrap()
|
||||
.into_direct()
|
||||
.unwrap();
|
||||
assert_eq!(logprobs, expected_prompt_logprobs());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decodes_big_endian_payloads() {
|
||||
let frames = vec![Bytes::from(encode_value(&output_wire_with_custom_fields(
|
||||
Some(Value::Array(vec![
|
||||
ndarray_value(">i4", &[1, 2], Value::Ext(3, vec![0, 0, 0, 1, 0, 0, 0, 2])),
|
||||
ndarray_value(
|
||||
">f4",
|
||||
&[1, 2],
|
||||
Value::Ext(3, vec![63, 128, 0, 0, 64, 0, 0, 0]),
|
||||
),
|
||||
ndarray_value(">i4", &[1], Value::Ext(3, vec![0, 0, 0, 3])),
|
||||
Value::Nil,
|
||||
])),
|
||||
None,
|
||||
)))];
|
||||
let decoded = decode_engine_core_outputs(&frames).unwrap();
|
||||
let logprobs = decoded.outputs[0].new_logprobs.clone().unwrap().into_direct().unwrap();
|
||||
assert_eq!(
|
||||
logprobs,
|
||||
Logprobs {
|
||||
positions: vec![PositionLogprobs {
|
||||
entries: vec![
|
||||
TokenLogprob {
|
||||
token_id: 1,
|
||||
logprob: 1.0,
|
||||
rank: 3,
|
||||
},
|
||||
TokenLogprob {
|
||||
token_id: 2,
|
||||
logprob: 2.0,
|
||||
rank: 1,
|
||||
},
|
||||
],
|
||||
}],
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_non_none_cu_num_generated_tokens() {
|
||||
let frames = vec![Bytes::from(encode_value(&output_wire_with_custom_fields(
|
||||
Some(Value::Array(vec![
|
||||
ndarray_value("<i8", &[1, 1], Value::Ext(3, vec![1, 0, 0, 0, 0, 0, 0, 0])),
|
||||
ndarray_value("<f4", &[1, 1], Value::Ext(3, vec![0, 0, 128, 63])),
|
||||
ndarray_value("<i8", &[1], Value::Ext(3, vec![1, 0, 0, 0, 0, 0, 0, 0])),
|
||||
Value::Array(vec![Value::from(0usize), Value::from(1usize)]),
|
||||
])),
|
||||
None,
|
||||
)))];
|
||||
|
||||
let error = decode_engine_core_outputs(&frames).unwrap_err();
|
||||
let crate::error::Error::ExtValueDecode { message } = &error else {
|
||||
panic!("expected ValueDecodeExt");
|
||||
};
|
||||
assert_eq!(
|
||||
message,
|
||||
"new_logprobs.cu_num_generated_tokens: expected None for per-request engine-core logprobs payload, got [0, 1]"
|
||||
);
|
||||
assert_eq!(
|
||||
error.to_string(),
|
||||
"messagepack ext value decode failed: new_logprobs.cu_num_generated_tokens: expected None for per-request engine-core logprobs payload, got [0, 1]"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
|
||||
|
||||
use crate::protocol::tensor::WireNdArray;
|
||||
|
||||
/// Python wire representation of `LogprobsLists` / `LogprobsTensors` before
|
||||
/// aux-frame references and raw-view payloads are resolved.
|
||||
///
|
||||
/// This mirrors the tuple shape emitted by Python engine-core so serde can
|
||||
/// first deserialize the raw wire payload before the Rust client converts it
|
||||
/// into semantic per-position logprobs records.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/outputs.py#L23-L56>
|
||||
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple)]
|
||||
pub struct WireLogprobs {
|
||||
/// Wire array with shape `[num_positions, max_num_logprobs + 1]`.
|
||||
pub logprob_token_ids: WireNdArray,
|
||||
/// Wire array with shape `[num_positions, max_num_logprobs + 1]`.
|
||||
pub logprobs: WireNdArray,
|
||||
/// Wire array with shape `[num_positions]`.
|
||||
///
|
||||
/// Python uses the field name `sampled_token_ranks` for sample logprobs and
|
||||
/// `selected_token_ranks` for prompt logprobs. Rust keeps one neutral field
|
||||
/// because both payloads share the same wire representation.
|
||||
pub token_ranks: WireNdArray,
|
||||
/// Preserved only for wire compatibility with batch-level Python tensors.
|
||||
/// Scheduler-sliced per-request outputs should emit `None` here, and
|
||||
/// the semantic Rust decoder rejects any other value.
|
||||
#[serde(default)]
|
||||
pub cu_num_generated_tokens: Option<Vec<usize>>,
|
||||
}
|
||||
@@ -0,0 +1,756 @@
|
||||
use std::any::type_name;
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
||||
use std::io::Cursor;
|
||||
|
||||
use bytes::Bytes;
|
||||
use rmpv::Value;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_default::DefaultFromSerde;
|
||||
use serde_repr::{Deserialize_repr, Serialize_repr};
|
||||
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
|
||||
use thiserror_ext::AsReport;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::protocol::logprobs::MaybeWireLogprobs;
|
||||
use crate::protocol::multimodal::MmFeatures;
|
||||
use crate::protocol::stats::{PrefillStats, SchedulerStats};
|
||||
|
||||
// TODO: This module currently mixes reusable frontend-facing semantic types
|
||||
// (for example `FinishReason`, `StopReason`, `RequestOutputKind`, and future
|
||||
// cleaned-up frontend sampling types) with engine-core-specific wire DTOs and
|
||||
// handshake/control messages. While the Rust frontend is still evolving
|
||||
// quickly, keep them co-located here for iteration speed. Once the higher-level
|
||||
// API boundary stabilizes, move the truly reusable semantic types into a
|
||||
// lower-level common crate and keep the engine transport/wire messages here.
|
||||
|
||||
/// Dynamic msgpack value used for schema positions that are preserved but not
|
||||
/// yet strongly typed in the early-stage Rust client.
|
||||
pub type OpaqueValue = Value;
|
||||
|
||||
fn default_opaque_value_nil() -> OpaqueValue {
|
||||
Value::Nil
|
||||
}
|
||||
|
||||
fn is_false(v: &bool) -> bool {
|
||||
!v
|
||||
}
|
||||
|
||||
mod classified_outputs;
|
||||
pub mod dtype;
|
||||
pub mod handshake;
|
||||
pub mod logprobs;
|
||||
pub mod multimodal;
|
||||
pub mod stats;
|
||||
pub mod tensor;
|
||||
pub use classified_outputs::{
|
||||
ClassifiedEngineCoreOutputs, DpControlMessage, RequestBatchOutputs, UtilityCallOutput,
|
||||
};
|
||||
pub use dtype::ModelDtype;
|
||||
pub use logprobs::decode_engine_core_outputs;
|
||||
|
||||
/// Request types are encoded as single-byte protocol constants so they can be
|
||||
/// sent over the ZMQ socket without an extra encoding step.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L217-L228>
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[repr(u8)]
|
||||
pub enum EngineCoreRequestType {
|
||||
Add = 0,
|
||||
Abort = 1,
|
||||
StartDpWave = 2,
|
||||
Utility = 3,
|
||||
}
|
||||
|
||||
impl EngineCoreRequestType {
|
||||
pub fn to_frame(self) -> Bytes {
|
||||
Bytes::from_static(match self {
|
||||
Self::Add => b"\x00",
|
||||
Self::Abort => b"\x01",
|
||||
Self::StartDpWave => b"\x02",
|
||||
Self::Utility => b"\x03",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Reason a request finished: stop, length, abort, error, or repetition.
|
||||
///
|
||||
/// This mirrors the Python enum and uses integer encoding for compact wire
|
||||
/// representation.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L41-L63>
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize_repr, Deserialize_repr)]
|
||||
#[repr(u8)]
|
||||
pub enum EngineCoreFinishReason {
|
||||
/// A stop string was emitted.
|
||||
Stop = 0,
|
||||
/// `max_tokens` or `max_model_len` was reached.
|
||||
Length = 1,
|
||||
/// The request was aborted by the client.
|
||||
Abort = 2,
|
||||
/// A retryable request-level internal error occurred.
|
||||
Error = 3,
|
||||
/// A repetitive token pattern was detected.
|
||||
Repetition = 4,
|
||||
}
|
||||
|
||||
/// Event types emitted by engine-core for one request.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L113-L118>
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize_repr, Deserialize_repr)]
|
||||
#[repr(u8)]
|
||||
pub enum EngineCoreEventType {
|
||||
Queued = 1,
|
||||
Scheduled = 2,
|
||||
Preempted = 3,
|
||||
}
|
||||
|
||||
/// A timestamped engine-core event associated with one request.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L121-L130>
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct EngineCoreEvent {
|
||||
pub r#type: EngineCoreEventType,
|
||||
pub timestamp: f64,
|
||||
}
|
||||
|
||||
/// Controls how intermediate outputs are returned to the frontend.
|
||||
///
|
||||
/// `Cumulative = 0` is intentionally not supported in Rust frontend.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/sampling_params.py#L146-L152>
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize_repr, Deserialize_repr)]
|
||||
#[repr(u8)]
|
||||
pub enum RequestOutputKind {
|
||||
/// Return only token deltas in each update.
|
||||
#[default]
|
||||
Delta = 1,
|
||||
/// Suppress intermediate updates and return only the final output.
|
||||
FinalOnly = 2,
|
||||
}
|
||||
|
||||
/// The stop reason associated with a finished output.
|
||||
///
|
||||
/// Python models this as the union-typed `stop_reason: int | str | None`
|
||||
/// field on `EngineCoreOutput`; the Rust client narrows it into a tagged enum.
|
||||
///
|
||||
/// Original Python field:
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L155>
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum StopReason {
|
||||
TokenId(u32),
|
||||
Text(String),
|
||||
}
|
||||
|
||||
/// Parameters for configuring structured outputs (guided decoding).
|
||||
///
|
||||
/// Exactly one constraint field (`json`, `regex`, `choice`, `grammar`,
|
||||
/// `json_object`, or `structural_tag`) should be set. The engine-core
|
||||
/// backend selects the appropriate grammar compiler based on which field
|
||||
/// is present.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/sampling_params.py#L36-L107>
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct StructuredOutputsParams {
|
||||
/// JSON schema (as a dict/object or JSON string) constraining the output.
|
||||
pub json: Option<serde_json::Value>,
|
||||
/// Regular expression the output must match.
|
||||
pub regex: Option<String>,
|
||||
/// List of allowed output strings (the model must produce one of these).
|
||||
pub choice: Option<Vec<String>>,
|
||||
/// Context-free grammar (in EBNF-like notation) the output must conform to.
|
||||
pub grammar: Option<String>,
|
||||
/// When `true`, output must be valid JSON (free-form, no schema).
|
||||
pub json_object: Option<bool>,
|
||||
/// Disable any additional whitespace in guided JSON output.
|
||||
#[serde(skip_serializing_if = "crate::protocol::is_false")]
|
||||
pub disable_any_whitespace: bool,
|
||||
/// Disable `additionalProperties` in JSON schema output.
|
||||
#[serde(skip_serializing_if = "crate::protocol::is_false")]
|
||||
pub disable_additional_properties: bool,
|
||||
/// Custom whitespace pattern for guided JSON output.
|
||||
pub whitespace_pattern: Option<String>,
|
||||
/// Structural tag configuration (JSON-encoded string).
|
||||
pub structural_tag: Option<String>,
|
||||
}
|
||||
|
||||
/// Engine-core-facing sampling parameters for text generation.
|
||||
///
|
||||
/// This is the normalized southbound subset used by the Rust frontend when it
|
||||
/// talks to Python engine-core over the wire. User-facing request semantics
|
||||
/// such as `stop` strings, `n`, `ignore_eos`, and output aggregation mode are
|
||||
/// intentionally handled by higher layers before values reach this DTO.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/sampling_params.py#L155-L291>
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct EngineCoreSamplingParams {
|
||||
/// Controls randomness. Lower values are more deterministic; zero means
|
||||
/// greedy sampling.
|
||||
pub temperature: f32,
|
||||
/// Cumulative probability threshold for nucleus sampling.
|
||||
pub top_p: f32,
|
||||
/// Maximum number of top tokens to consider. `0` means all tokens.
|
||||
pub top_k: u32,
|
||||
/// Random seed used by the sampler when present.
|
||||
pub seed: Option<i64>,
|
||||
/// Maximum number of tokens to generate per output sequence.
|
||||
pub max_tokens: u32,
|
||||
/// Minimum number of tokens to generate before EOS or stop-token handling.
|
||||
pub min_tokens: u32,
|
||||
/// Number of log probabilities to return per generated token.
|
||||
///
|
||||
/// `None` disables sample logprobs. `-1` requests the full vocabulary.
|
||||
pub logprobs: Option<i32>,
|
||||
/// Number of log probabilities to return per prompt token.
|
||||
///
|
||||
/// `None` disables prompt logprobs. `-1` requests the full vocabulary.
|
||||
pub prompt_logprobs: Option<i32>,
|
||||
/// Minimum probability threshold for token sampling.
|
||||
pub min_p: f32,
|
||||
/// Frequency penalty applied by the sampler.
|
||||
pub frequency_penalty: f32,
|
||||
/// Presence penalty applied by the sampler.
|
||||
pub presence_penalty: f32,
|
||||
/// Repetition penalty applied by the sampler.
|
||||
pub repetition_penalty: f32,
|
||||
/// Token IDs that stop generation.
|
||||
pub stop_token_ids: Vec<u32>,
|
||||
/// Primary EOS token ID used by engine-core's dedicated EOS stop path.
|
||||
///
|
||||
/// This mirrors Python's internal `_eos_token_id` field and is derived by
|
||||
/// the frontend from tokenizer/model metadata rather than supplied directly
|
||||
/// by end users.
|
||||
#[serde(rename = "_eos_token_id")]
|
||||
pub eos_token_id: Option<u32>,
|
||||
/// Complete stop-token set used by engine-core for `min_tokens` masking.
|
||||
///
|
||||
/// This mirrors Python's internal `_all_stop_token_ids` field and should
|
||||
/// contain explicit `stop_token_ids` plus any frontend-derived EOS token
|
||||
/// IDs.
|
||||
#[serde(rename = "_all_stop_token_ids")]
|
||||
pub all_stop_token_ids: BTreeSet<u32>,
|
||||
/// Logit biases to apply during sampling.
|
||||
/// Keys are token IDs
|
||||
#[serde(default)]
|
||||
pub logit_bias: Option<HashMap<u32, f32>>,
|
||||
/// Restrict output to these token IDs only.
|
||||
#[serde(default)]
|
||||
pub allowed_token_ids: Option<Vec<u32>>,
|
||||
/// Tokenized bad words to avoid during generation.
|
||||
#[serde(default, rename = "_bad_words_token_ids")]
|
||||
pub bad_words_token_ids: Option<Vec<Vec<u32>>>,
|
||||
/// Parameters for configuring structured outputs (guided decoding).
|
||||
#[serde(default)]
|
||||
pub structured_outputs: Option<StructuredOutputsParams>,
|
||||
/// Specific token IDs for which log probabilities should be returned at
|
||||
/// each position.
|
||||
///
|
||||
/// When set, the engine returns logprobs for exactly these tokens in
|
||||
/// addition to the sampled/scored token. Mutually exclusive with the
|
||||
/// `logprobs` count field in practice.
|
||||
#[serde(default)]
|
||||
pub logprob_token_ids: Option<Vec<u32>>,
|
||||
/// If `Some(true)`, the request will not attempt to read from the prefix
|
||||
/// cache; newly computed blocks may still populate the cache. `None`
|
||||
/// defers to engine-core defaults.
|
||||
#[serde(default)]
|
||||
pub skip_reading_prefix_cache: Option<bool>,
|
||||
/// Additional request parameters for custom extensions (from `vllm_xargs`).
|
||||
#[serde(default)]
|
||||
pub extra_args: Option<HashMap<String, serde_json::Value>>,
|
||||
}
|
||||
|
||||
impl EngineCoreSamplingParams {
|
||||
/// Constructs a default sampling params for testing purposes only.
|
||||
pub fn for_test() -> Self {
|
||||
Self {
|
||||
temperature: 1.0,
|
||||
top_p: 1.0,
|
||||
top_k: 0,
|
||||
seed: None,
|
||||
max_tokens: 65536,
|
||||
min_tokens: 0,
|
||||
logprobs: None,
|
||||
prompt_logprobs: None,
|
||||
min_p: 0.0,
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
repetition_penalty: 1.0,
|
||||
stop_token_ids: Vec::new(),
|
||||
eos_token_id: None,
|
||||
all_stop_token_ids: BTreeSet::new(),
|
||||
logit_bias: None,
|
||||
allowed_token_ids: None,
|
||||
bad_words_token_ids: None,
|
||||
structured_outputs: None,
|
||||
logprob_token_ids: None,
|
||||
skip_reading_prefix_cache: None,
|
||||
extra_args: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Engine-core add-request payload sent from frontend to engine.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/3f5bd482f5c1a5dbdffbbf68d624e20bb7032013/vllm/v1/engine/__init__.py#L80-L129>
|
||||
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple, DefaultFromSerde)]
|
||||
pub struct EngineCoreRequest {
|
||||
pub request_id: String,
|
||||
pub prompt_token_ids: Option<Vec<u32>>,
|
||||
/// Multimodal features attached to the request.
|
||||
pub mm_features: Option<MmFeatures>,
|
||||
pub sampling_params: Option<EngineCoreSamplingParams>,
|
||||
/// Pooling parameters are preserved in the schema but not yet strongly
|
||||
/// typed.
|
||||
pub pooling_params: Option<OpaqueValue>,
|
||||
pub arrival_time: f64,
|
||||
#[serde(default)]
|
||||
pub lora_request: Option<OpaqueValue>,
|
||||
#[serde(default)]
|
||||
pub cache_salt: Option<String>,
|
||||
#[serde(default)]
|
||||
pub data_parallel_rank: Option<u32>,
|
||||
/// Unsupported in the first-stage Rust client because Python uses a custom
|
||||
/// tensor/aux-frame encoding path for this field.
|
||||
#[serde(default)]
|
||||
pub prompt_embeds: Option<OpaqueValue>,
|
||||
/// Per-position mask for mixed-mode inputs (e.g. chat completion with
|
||||
/// `prompt_embeds` content parts). `Some(true)` means real token id;
|
||||
/// `Some(false)` means the position uses a pre-computed entry from
|
||||
/// `prompt_embeds`. `None` for pure-tokens and pure-embeds requests.
|
||||
#[serde(default)]
|
||||
pub prompt_is_token_ids: Option<Vec<bool>>,
|
||||
/// Index of the client, used to ensure outputs are sent back to the same
|
||||
/// client when scaling out the frontend.
|
||||
#[serde(default)]
|
||||
pub client_index: u32,
|
||||
/// In DP mode, indicates which wave this request is expected to belong to.
|
||||
#[serde(default)]
|
||||
pub current_wave: u32,
|
||||
#[serde(default)]
|
||||
pub priority: i32,
|
||||
#[serde(default)]
|
||||
pub trace_headers: Option<BTreeMap<String, String>>,
|
||||
#[serde(default)]
|
||||
pub resumable: bool,
|
||||
/// Original user-provided request ID, used for output reporting and aborts.
|
||||
#[serde(default)]
|
||||
pub external_req_id: Option<String>,
|
||||
#[serde(default)]
|
||||
pub reasoning_ended: Option<bool>,
|
||||
/// Opaque reasoning-parser kwargs forwarded from the frontend to the
|
||||
/// structured-output backend.
|
||||
#[serde(default)]
|
||||
pub reasoning_parser_kwargs: Option<OpaqueValue>,
|
||||
/// If `true`, the request should be added to the scheduler's waiting queue
|
||||
/// and immediately aborted, so connector-side cleanup runs via the
|
||||
/// standard `request_finished` hook.
|
||||
#[serde(default)]
|
||||
pub abort_immediately: bool,
|
||||
}
|
||||
|
||||
impl EngineCoreRequest {
|
||||
/// Validate fields intentionally not supported in the first-stage client.
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
if self.prompt_embeds.is_some() {
|
||||
return Err(Error::UnsupportedField {
|
||||
context: "EngineCoreRequest",
|
||||
field: "prompt_embeds",
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Engine-core utility call payload sent from frontend to engine.
|
||||
///
|
||||
/// Original Python payload shape:
|
||||
/// `(client_index, call_id, method_name, args)`
|
||||
#[derive(Debug, Clone, PartialEq, Serialize_tuple)]
|
||||
pub struct EngineCoreUtilityRequest {
|
||||
pub client_index: u32,
|
||||
pub call_id: i64,
|
||||
pub method_name: String,
|
||||
pub args: OpaqueValue,
|
||||
}
|
||||
|
||||
impl EngineCoreUtilityRequest {
|
||||
/// Create a new utility request with the given strongly typed arguments,
|
||||
/// encoding them into the expected msgpack value format.
|
||||
pub fn new<T>(
|
||||
client_index: u32,
|
||||
call_id: i64,
|
||||
method_name: impl Into<String>,
|
||||
args: T,
|
||||
) -> Result<Self>
|
||||
where
|
||||
T: Serialize + std::fmt::Debug,
|
||||
{
|
||||
let args = rmpv::ext::to_value(&args).map_err(|error| Error::Encode {
|
||||
target_type: type_name::<T>(),
|
||||
message: format!(
|
||||
"failed to encode utility args `{args:?}`: {}",
|
||||
error.to_report_string()
|
||||
),
|
||||
})?;
|
||||
let args = match args {
|
||||
Value::Nil => Value::Array(Vec::new()),
|
||||
other => other,
|
||||
};
|
||||
|
||||
Ok(Self {
|
||||
client_index,
|
||||
call_id,
|
||||
method_name: method_name.into(),
|
||||
args,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Engine-core output for a single request.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/d3af8c18317c0dc008d42e4367fbb9045cfb7bf6/vllm/v1/engine/__init__.py#L154-L184>
|
||||
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple, DefaultFromSerde)]
|
||||
pub struct EngineCoreOutput {
|
||||
pub request_id: String,
|
||||
pub new_token_ids: Vec<u32>,
|
||||
/// Decoded sample logprobs for the newly generated positions in this
|
||||
/// output.
|
||||
#[serde(default)]
|
||||
pub new_logprobs: Option<MaybeWireLogprobs>,
|
||||
/// Decoded prompt logprobs for the scored prompt positions emitted in this
|
||||
/// output.
|
||||
#[serde(default)]
|
||||
pub new_prompt_logprobs_tensors: Option<MaybeWireLogprobs>,
|
||||
#[serde(default)]
|
||||
pub pooling_output: Option<OpaqueValue>,
|
||||
#[serde(default)]
|
||||
pub finish_reason: Option<EngineCoreFinishReason>,
|
||||
#[serde(default)]
|
||||
pub stop_reason: Option<StopReason>,
|
||||
#[serde(default)]
|
||||
pub events: Option<Vec<EngineCoreEvent>>,
|
||||
#[serde(default)]
|
||||
pub kv_transfer_params: Option<serde_json::Value>,
|
||||
#[serde(default)]
|
||||
pub trace_headers: Option<OpaqueValue>,
|
||||
/// Breakdown of the scheduled prefill computation, set on the first output
|
||||
/// of a newly scheduled prefill and elided for subsequent decode outputs.
|
||||
#[serde(default)]
|
||||
pub prefill_stats: Option<PrefillStats>,
|
||||
#[serde(default)]
|
||||
pub routed_experts: Option<OpaqueValue>,
|
||||
/// Number of NaNs seen in logits. Values above zero indicate corruption.
|
||||
#[serde(default)]
|
||||
pub num_nans_in_logits: u32,
|
||||
}
|
||||
|
||||
impl EngineCoreOutput {
|
||||
/// Returns whether this output is terminal for the request.
|
||||
pub fn finished(&self) -> bool {
|
||||
self.finish_reason.is_some()
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of a utility call.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L174-L183>
|
||||
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple, DefaultFromSerde)]
|
||||
pub struct UtilityOutput {
|
||||
pub call_id: i64,
|
||||
/// Non-`None` implies the call failed and `result` should be ignored.
|
||||
#[serde(default)]
|
||||
pub failure_message: Option<String>,
|
||||
#[serde(default)]
|
||||
pub result: Option<UtilityResultEnvelope>,
|
||||
}
|
||||
|
||||
/// Python `UtilityResult` wrapper carried inside `UtilityOutput.result`.
|
||||
///
|
||||
/// Upstream reference:
|
||||
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/serial_utils.py#L178-L185>
|
||||
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple)]
|
||||
pub struct UtilityResultEnvelope {
|
||||
/// Recursive type information encoded on Python side, serving as the hint
|
||||
/// for deserialization. We don't care it here as in Rust frontend all
|
||||
/// utility calls are strongly-typed.
|
||||
#[serde(default)]
|
||||
type_info: Option<OpaqueValue>,
|
||||
/// The actual utility result.
|
||||
#[serde(default = "default_opaque_value_nil")]
|
||||
result: OpaqueValue,
|
||||
}
|
||||
|
||||
impl UtilityResultEnvelope {
|
||||
/// Create a utility result envelope without type information.
|
||||
pub fn without_type_info(result: OpaqueValue) -> Self {
|
||||
Self {
|
||||
type_info: None,
|
||||
result,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl UtilityOutput {
|
||||
/// Decode the typed result of a utility call.
|
||||
pub fn into_typed_result<T>(self, method: &str) -> Result<T>
|
||||
where
|
||||
T: serde::de::DeserializeOwned,
|
||||
{
|
||||
if let Some(message) = self.failure_message {
|
||||
return Err(Error::UtilityCallFailed {
|
||||
method: method.to_string(),
|
||||
call_id: self.call_id,
|
||||
message,
|
||||
});
|
||||
}
|
||||
|
||||
let result = self.result.map(|e| e.result).unwrap_or(Value::Nil);
|
||||
|
||||
rmpv::ext::from_value(result).map_err(|error| Error::UtilityResultDecode {
|
||||
method: method.to_string(),
|
||||
call_id: self.call_id,
|
||||
message: error.to_report_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Batch of engine-core outputs returned to a frontend client.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L186-L214>
|
||||
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple, DefaultFromSerde)]
|
||||
pub struct EngineCoreOutputs {
|
||||
#[serde(default)]
|
||||
pub engine_index: u32,
|
||||
/// Outputs grouped for this client in the current engine tick.
|
||||
#[serde(default)]
|
||||
pub outputs: Vec<EngineCoreOutput>,
|
||||
#[serde(default)]
|
||||
pub scheduler_stats: Option<Box<SchedulerStats>>,
|
||||
#[serde(default)]
|
||||
pub timestamp: f64,
|
||||
#[serde(default)]
|
||||
pub utility_output: Option<UtilityOutput>,
|
||||
#[serde(default)]
|
||||
pub finished_requests: Option<BTreeSet<String>>,
|
||||
/// In DP mode, signals that the current wave finished and engines are
|
||||
/// paused.
|
||||
#[serde(default)]
|
||||
pub wave_complete: Option<u32>,
|
||||
/// In DP mode, signals that a request arrived for an old wave and the next
|
||||
/// wave needs to start in other engines.
|
||||
#[serde(default)]
|
||||
pub start_wave: Option<u32>,
|
||||
}
|
||||
|
||||
/// Encode a Rust value into msgpack using the protocol crate's serde model.
|
||||
pub fn encode_msgpack<T>(value: &T) -> Result<Vec<u8>>
|
||||
where
|
||||
T: Serialize + std::fmt::Debug,
|
||||
{
|
||||
rmp_serde::to_vec_named(value).map_err(|error| Error::Encode {
|
||||
target_type: type_name::<T>(),
|
||||
message: format!(
|
||||
"failed to encode value `{:?}`: {}",
|
||||
value,
|
||||
error.to_report_string()
|
||||
),
|
||||
})
|
||||
}
|
||||
|
||||
/// Decode a msgpack payload into a strongly typed protocol value, with enhanced
|
||||
/// error reporting.
|
||||
pub fn decode_msgpack<T>(bytes: &[u8]) -> Result<T>
|
||||
where
|
||||
T: for<'de> Deserialize<'de>,
|
||||
{
|
||||
fn decode_value_preview(bytes: &[u8]) -> String {
|
||||
match decode_value(bytes) {
|
||||
Ok(value) => format!("{value}"),
|
||||
Err(error) => format!("<value decode failed: {error}>"),
|
||||
}
|
||||
}
|
||||
|
||||
rmp_serde::from_slice(bytes).map_err(|error| Error::Decode {
|
||||
target_type: type_name::<T>(),
|
||||
message: format!("{error}; value fallback: {}", decode_value_preview(bytes)),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn decode_value(bytes: &[u8]) -> Result<Value> {
|
||||
Ok(rmpv::decode::read_value(&mut Cursor::new(bytes))?)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn utility_result_value<T>(value: T) -> UtilityResultEnvelope
|
||||
where
|
||||
T: Serialize,
|
||||
{
|
||||
UtilityResultEnvelope::without_type_info(rmpv::ext::to_value(value).unwrap())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn engine_core_request_serializes_as_full_array() {
|
||||
let request = EngineCoreRequest {
|
||||
request_id: "req-1".to_string(),
|
||||
prompt_token_ids: Some(vec![1, 2, 3]),
|
||||
sampling_params: Some(EngineCoreSamplingParams {
|
||||
max_tokens: 8,
|
||||
..EngineCoreSamplingParams::for_test()
|
||||
}),
|
||||
arrival_time: 1234.5,
|
||||
client_index: 7,
|
||||
..EngineCoreRequest::default()
|
||||
};
|
||||
|
||||
let encoded = encode_msgpack(&request).unwrap();
|
||||
let value = decode_value(&encoded).unwrap();
|
||||
let array = match value {
|
||||
Value::Array(array) => array,
|
||||
other => panic!("expected array, got {other:?}"),
|
||||
};
|
||||
|
||||
assert_eq!(array.len(), 20);
|
||||
assert_eq!(array[0], Value::from("req-1"));
|
||||
assert_eq!(array[2], Value::Nil);
|
||||
assert_eq!(array[4], Value::Nil);
|
||||
assert_eq!(array[10], Value::Nil);
|
||||
assert_eq!(array[11], Value::from(7));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn engine_core_outputs_roundtrip_finished_fields() {
|
||||
let outputs = EngineCoreOutputs {
|
||||
outputs: vec![EngineCoreOutput {
|
||||
request_id: "req-1".to_string(),
|
||||
new_token_ids: vec![42],
|
||||
new_logprobs: None,
|
||||
new_prompt_logprobs_tensors: None,
|
||||
pooling_output: None,
|
||||
finish_reason: Some(EngineCoreFinishReason::Length),
|
||||
stop_reason: Some(StopReason::Text("stop".to_string())),
|
||||
events: None,
|
||||
kv_transfer_params: None,
|
||||
trace_headers: None,
|
||||
prefill_stats: None,
|
||||
routed_experts: None,
|
||||
num_nans_in_logits: 0,
|
||||
}],
|
||||
finished_requests: Some(BTreeSet::from(["req-1".to_string()])),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let encoded = encode_msgpack(&outputs).unwrap();
|
||||
let decoded: EngineCoreOutputs = decode_msgpack(&encoded).unwrap();
|
||||
|
||||
assert_eq!(decoded.outputs.len(), 1);
|
||||
assert_eq!(
|
||||
decoded.outputs[0].finish_reason,
|
||||
Some(EngineCoreFinishReason::Length)
|
||||
);
|
||||
assert_eq!(
|
||||
decoded.finished_requests,
|
||||
Some(BTreeSet::from(["req-1".to_string()]))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn utility_request_serializes_as_tuple_payload() {
|
||||
let request = EngineCoreUtilityRequest::new(7, 42, "is_sleeping", ()).unwrap();
|
||||
|
||||
let encoded = encode_msgpack(&request).unwrap();
|
||||
let value = decode_value(&encoded).unwrap();
|
||||
let array = match value {
|
||||
Value::Array(array) => array,
|
||||
other => panic!("expected utility request array, got {other:?}"),
|
||||
};
|
||||
|
||||
assert_eq!(array.len(), 4);
|
||||
assert_eq!(array[0], Value::from(7));
|
||||
assert_eq!(array[1], Value::from(42));
|
||||
assert_eq!(array[2], Value::from("is_sleeping"));
|
||||
assert_eq!(array[3], Value::Array(Vec::new()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn utility_output_decodes_typed_result() {
|
||||
let output = UtilityOutput {
|
||||
call_id: 9,
|
||||
failure_message: None,
|
||||
result: Some(utility_result_value(true)),
|
||||
};
|
||||
|
||||
assert!(output.into_typed_result::<bool>("is_sleeping").unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn utility_output_reports_failure_message() {
|
||||
let error = UtilityOutput {
|
||||
call_id: 9,
|
||||
failure_message: Some("boom".to_string()),
|
||||
result: None,
|
||||
}
|
||||
.into_typed_result::<bool>("is_sleeping")
|
||||
.unwrap_err();
|
||||
|
||||
assert!(matches!(
|
||||
error,
|
||||
Error::UtilityCallFailed {
|
||||
method,
|
||||
call_id,
|
||||
message
|
||||
} if method == "is_sleeping" && call_id == 9 && message == "boom"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn utility_output_decodes_missing_result_as_unit() {
|
||||
UtilityOutput {
|
||||
call_id: 3,
|
||||
failure_message: None,
|
||||
result: None,
|
||||
}
|
||||
.into_typed_result::<()>("reset_mm_cache")
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn utility_output_decodes_nil_result_as_unit() {
|
||||
UtilityOutput {
|
||||
call_id: 4,
|
||||
failure_message: None,
|
||||
result: Some(UtilityResultEnvelope::without_type_info(Value::Nil)),
|
||||
}
|
||||
.into_typed_result::<()>("sleep")
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_msgpack_includes_type_name_and_value_fallback() {
|
||||
let error = decode_msgpack::<u64>(
|
||||
&rmp_serde::to_vec_named(&BTreeMap::from([("status", "READY")])).unwrap(),
|
||||
)
|
||||
.unwrap_err();
|
||||
|
||||
expect_test::expect![[r#"messagepack decode failed for u64: wrong msgpack marker FixMap(1); value fallback: {"status": "READY"}"#]].assert_eq(&error.to_report_string());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,282 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
|
||||
|
||||
use super::tensor::WireTensor;
|
||||
|
||||
/// Multimodal feature payload accepted from higher-level frontend code.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/v1/engine/__init__.py#L88>
|
||||
pub type MmFeatures = Vec<MmFeatureSpec>;
|
||||
|
||||
/// Represents a single multimodal input with its processed data and metadata.
|
||||
///
|
||||
/// Used to track multimodal data through processing and caching. A request
|
||||
/// containing multiple multimodal items will have one `MmFeatureSpec`
|
||||
/// per item.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/multimodal/inputs.py#L301-L332>
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct MmFeatureSpec {
|
||||
/// Represents multimodal data for this feature.
|
||||
///
|
||||
/// Can be `None` if the item is cached, to skip IPC between API server
|
||||
/// and engine core processes.
|
||||
pub data: Option<MmKwargsItem>,
|
||||
|
||||
/// The input modality, e.g., `"image"`, `"audio"`, `"video"`.
|
||||
pub modality: String,
|
||||
|
||||
/// The hash for caching encoder outputs (with LoRA prefix if applicable).
|
||||
pub identifier: String,
|
||||
|
||||
/// The location of the `modality` tokens corresponding to this item
|
||||
/// in the prompt, e.g., `PlaceholderRange(offset=2, length=336)`.
|
||||
pub mm_position: PlaceholderRange,
|
||||
|
||||
/// The hash for caching processor outputs (without LoRA prefix).
|
||||
#[serde(default)]
|
||||
pub mm_hash: Option<String>,
|
||||
}
|
||||
|
||||
/// Placeholder location information for multi-modal data.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/multimodal/inputs.py#L118-L145>
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct PlaceholderRange {
|
||||
/// The start index of the placeholder in the prompt.
|
||||
pub offset: usize,
|
||||
|
||||
/// The length of the placeholder.
|
||||
pub length: usize,
|
||||
|
||||
/// A boolean mask of shape `(length,)` indicating which positions
|
||||
/// between `offset` and `offset + length` to assign embeddings to.
|
||||
/// `None` means all positions.
|
||||
#[serde(default)]
|
||||
pub is_embed: Option<WireTensor>,
|
||||
}
|
||||
|
||||
/// A dictionary of processed keyword arguments to pass to the model,
|
||||
/// corresponding to a single item in `MultiModalDataItems`.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/multimodal/inputs.py#L854-L871>
|
||||
pub type MmKwargsItem = BTreeMap<String, MmFieldElem>;
|
||||
|
||||
/// Represents a processed keyword argument to pass to a model for a
|
||||
/// `MmKwargsItem`.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/multimodal/inputs.py#L348-L369>
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct MmFieldElem {
|
||||
/// The processed value of this field in `MmKwargsItem`, i.e. the
|
||||
/// keyword argument value to be passed to the model.
|
||||
///
|
||||
/// It may be set to `None` if it is determined that the item is cached
|
||||
/// in `EngineCore`.
|
||||
pub data: Option<MmKwargValue>,
|
||||
|
||||
/// Defines how to combine this field's processed values with others in
|
||||
/// order to batch multi-modal items together for model inference.
|
||||
pub field: MmField,
|
||||
}
|
||||
|
||||
/// Processed multimodal keyword argument value.
|
||||
///
|
||||
/// Original Python definition (`NestedTensors`) and wire encoding:
|
||||
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/multimodal/inputs.py#L218-L226>
|
||||
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/v1/serial_utils.py#L292-L299>
|
||||
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/v1/serial_utils.py#L456-L465>
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum MmKwargValue {
|
||||
Tensor(WireTensor),
|
||||
Int(i64),
|
||||
Float(f64),
|
||||
List(Vec<MmKwargValue>),
|
||||
}
|
||||
|
||||
/// Defines how to interpret tensor data belonging to a keyword argument for
|
||||
/// `MultiModalKwargsItems`, and vice versa.
|
||||
///
|
||||
/// Original Python definitions and wire encoding:
|
||||
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/multimodal/inputs.py#L385-L630>
|
||||
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/v1/serial_utils.py#L301-L310>
|
||||
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/v1/serial_utils.py#L440-L454>
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(try_from = "MmFieldWire", into = "MmFieldWire")]
|
||||
pub enum MmField {
|
||||
Batched(MmBatchedField),
|
||||
Flat(MmFlatField),
|
||||
Shared(MmSharedField),
|
||||
}
|
||||
|
||||
/// Info: `MultiModalFieldConfig.batched`.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/multimodal/inputs.py#L385-L502>
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MmBatchedField {
|
||||
/// If `True`, then this field is excluded from being moved to the
|
||||
/// accelerator when multimodal items are grouped and batched.
|
||||
pub keep_on_cpu: bool,
|
||||
}
|
||||
|
||||
/// Info: `MultiModalFieldConfig.flat` and
|
||||
/// `MultiModalFieldConfig.flat_from_sizes`.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/multimodal/inputs.py#L385-L397>
|
||||
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/multimodal/inputs.py#L505-L603>
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MmFlatField {
|
||||
/// For each multi-modal item, a slice (`dim=0`) or a tuple of slices
|
||||
/// (`dim>0`) that is used to extract the data corresponding to it.
|
||||
pub slices: Vec<MmSlice>,
|
||||
|
||||
/// The dimension to extract data, default to 0.
|
||||
pub dim: i32,
|
||||
|
||||
/// If `True`, then this field is excluded from being moved to the
|
||||
/// accelerator when multimodal items are grouped and batched.
|
||||
pub keep_on_cpu: bool,
|
||||
}
|
||||
|
||||
/// Info: `MultiModalFieldConfig.shared`.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/multimodal/inputs.py#L385-L397>
|
||||
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/multimodal/inputs.py#L606-L630>
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MmSharedField {
|
||||
pub batch_size: usize,
|
||||
|
||||
/// If `True`, then this field is excluded from being moved to the
|
||||
/// accelerator when multimodal items are grouped and batched.
|
||||
pub keep_on_cpu: bool,
|
||||
}
|
||||
|
||||
/// Python slice encoded as `(start, stop, step)`.
|
||||
///
|
||||
/// Original Python wire encoding:
|
||||
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/v1/serial_utils.py#L199-L204>
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize_tuple, Deserialize_tuple)]
|
||||
pub struct SliceSpec {
|
||||
pub start: Option<isize>,
|
||||
pub stop: Option<isize>,
|
||||
pub step: Option<isize>,
|
||||
}
|
||||
|
||||
/// A single slice or a tuple of slices used by `MmFlatField`.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum MmSlice {
|
||||
Slice(SliceSpec),
|
||||
Slices(Vec<SliceSpec>),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize_tuple, Deserialize_tuple)]
|
||||
struct MmFieldWire {
|
||||
name: String,
|
||||
inner: MmFieldWireInner,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
enum MmFieldWireInner {
|
||||
Batched(MmBatchedField),
|
||||
Flat(MmFlatField),
|
||||
Shared(MmSharedField),
|
||||
}
|
||||
|
||||
impl TryFrom<MmFieldWire> for MmField {
|
||||
type Error = String;
|
||||
|
||||
fn try_from(value: MmFieldWire) -> Result<Self, Self::Error> {
|
||||
match (value.name.as_str(), value.inner) {
|
||||
("batched", MmFieldWireInner::Batched(kwargs)) => Ok(Self::Batched(kwargs)),
|
||||
("flat", MmFieldWireInner::Flat(kwargs)) => Ok(Self::Flat(kwargs)),
|
||||
("shared", MmFieldWireInner::Shared(kwargs)) => Ok(Self::Shared(kwargs)),
|
||||
(name, _) => Err(format!(
|
||||
"mismatched or unknown multimodal field factory {name:?}"
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<MmField> for MmFieldWire {
|
||||
fn from(value: MmField) -> Self {
|
||||
match value {
|
||||
MmField::Batched(kwargs) => Self {
|
||||
name: "batched".to_string(),
|
||||
inner: MmFieldWireInner::Batched(kwargs),
|
||||
},
|
||||
MmField::Flat(kwargs) => Self {
|
||||
name: "flat".to_string(),
|
||||
inner: MmFieldWireInner::Flat(kwargs),
|
||||
},
|
||||
MmField::Shared(kwargs) => Self {
|
||||
name: "shared".to_string(),
|
||||
inner: MmFieldWireInner::Shared(kwargs),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::io::Cursor;
|
||||
|
||||
use rmpv::Value;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn encode_value<T: Serialize>(value: &T) -> Value {
|
||||
let bytes = rmp_serde::to_vec_named(value).expect("encode value");
|
||||
rmpv::decode::read_value(&mut Cursor::new(bytes)).expect("decode value")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multimodal_field_serializes_to_python_factory_tuple() {
|
||||
let field = MmField::Flat(MmFlatField {
|
||||
slices: vec![MmSlice::Slice(SliceSpec {
|
||||
start: Some(0),
|
||||
stop: Some(1200),
|
||||
step: None,
|
||||
})],
|
||||
dim: 0,
|
||||
keep_on_cpu: false,
|
||||
});
|
||||
|
||||
let value = encode_value(&field);
|
||||
let Value::Array(items) = value else {
|
||||
panic!("field should encode as tuple array");
|
||||
};
|
||||
assert_eq!(items.len(), 2);
|
||||
assert_eq!(items[0].as_str(), Some("flat"));
|
||||
|
||||
let Value::Map(kwargs) = &items[1] else {
|
||||
panic!("field kwargs should encode as map");
|
||||
};
|
||||
assert!(kwargs.iter().any(|(key, _)| key.as_str() == Some("slices")));
|
||||
assert!(kwargs.iter().any(|(key, _)| key.as_str() == Some("dim")));
|
||||
assert!(kwargs.iter().any(|(key, _)| key.as_str() == Some("keep_on_cpu")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multimodal_field_round_trips_python_factory_tuple() {
|
||||
let field = MmField::Batched(MmBatchedField { keep_on_cpu: true });
|
||||
let encoded = rmp_serde::to_vec_named(&field).expect("encode field");
|
||||
let decoded: MmField = rmp_serde::from_slice(&encoded).expect("decode field");
|
||||
assert_eq!(decoded, field);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::protocol::OpaqueValue;
|
||||
|
||||
/// Stores cache hit statistics.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/metrics/stats.py#L18-L32>
|
||||
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct BaseCacheStats {
|
||||
/// Whether the cache was reset.
|
||||
pub reset: bool,
|
||||
/// The number of requests in this update.
|
||||
pub requests: u64,
|
||||
/// The number of queries in these requests.
|
||||
pub queries: u64,
|
||||
/// The number of hits in these requests.
|
||||
pub hits: u64,
|
||||
}
|
||||
|
||||
/// Stores prefix cache hit statistics.
|
||||
/// - `reset`: Whether `reset_prefix_cache` was invoked.
|
||||
/// - `queries`: Refers to the number of tokens that were queried.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/metrics/stats.py#L114-L143>
|
||||
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct PrefixCacheStats {
|
||||
/// Embedded base cache counters and reset flag.
|
||||
#[serde(flatten)]
|
||||
pub base: BaseCacheStats,
|
||||
/// The number of previously preempted requests in this update.
|
||||
pub preempted_requests: u64,
|
||||
/// The `queries` number for preempted requests.
|
||||
pub preempted_queries: u64,
|
||||
/// The `hits` number for preempted requests.
|
||||
pub preempted_hits: u64,
|
||||
}
|
||||
|
||||
/// Single KV cache block eviction sample.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/metrics/stats.py#L161-L167>
|
||||
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct KvCacheEvictionEvent {
|
||||
/// Lifetime from allocation to eviction.
|
||||
pub lifetime_seconds: f64,
|
||||
/// Idle time observed before eviction.
|
||||
pub idle_seconds: f64,
|
||||
/// Time gaps between consecutive accesses before eviction.
|
||||
pub reuse_gaps_seconds: Vec<f64>,
|
||||
}
|
||||
|
||||
/// Per-step iteration decoding stats from scheduler.
|
||||
///
|
||||
/// Each scheduler step, statistics on spec decoding performance are aggregated
|
||||
/// across requests by the scheduler and returned to the frontend in
|
||||
/// `EngineCoreOutputs -> SchedulerStats`.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/spec_decode/metrics.py#L16-L44>
|
||||
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct SpecDecodingStats {
|
||||
/// Configured speculative token count for this scheduler.
|
||||
pub num_spec_tokens: u64,
|
||||
/// Number of drafted speculative decoding attempts.
|
||||
pub num_drafts: u64,
|
||||
/// Number of drafted tokens.
|
||||
pub num_draft_tokens: u64,
|
||||
/// Number of accepted drafted tokens.
|
||||
pub num_accepted_tokens: u64,
|
||||
/// Accepted drafted tokens counted by draft position.
|
||||
pub num_accepted_tokens_per_pos: Vec<u64>,
|
||||
}
|
||||
|
||||
/// Breakdown of a scheduled prefill computation.
|
||||
///
|
||||
/// Python models this as a plain `@dataclass`, so it is serialized by msgspec
|
||||
/// as a map (named fields) rather than in the array-like form used by
|
||||
/// `EngineCoreOutput` itself.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/d3af8c18317c0dc008d42e4367fbb9045cfb7bf6/vllm/v1/metrics/stats.py#L242-L273>
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct PrefillStats {
|
||||
/// Total number of tokens to be prefilled.
|
||||
#[serde(default)]
|
||||
pub num_prompt_tokens: u32,
|
||||
/// Tokens to be prefilled locally (actual compute work).
|
||||
#[serde(default)]
|
||||
pub num_computed_tokens: u32,
|
||||
/// Tokens to be prefilled without actual compute work.
|
||||
#[serde(default)]
|
||||
pub num_cached_tokens: u32,
|
||||
/// Tokens to be prefilled from local prefix cache.
|
||||
#[serde(default)]
|
||||
pub num_local_cached_tokens: u32,
|
||||
/// Tokens to be prefilled from external KV transfer.
|
||||
#[serde(default)]
|
||||
pub num_external_cached_tokens: u32,
|
||||
}
|
||||
|
||||
/// Stats for debugging the metrics calculation.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/metrics/perf.py#L46-L55>
|
||||
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct DebugPerfStats {
|
||||
/// Time spent calculating these stats.
|
||||
pub calc_duration: f64,
|
||||
/// Number of prefill requests included in the sampled batch.
|
||||
pub num_prefill_requests: u64,
|
||||
/// Number of decode requests included in the sampled batch.
|
||||
pub num_decode_requests: u64,
|
||||
/// Optional execution-context breakdown used for debugging.
|
||||
pub context_breakdown: Option<BTreeMap<String, u64>>,
|
||||
/// Optional per-component FLOPs breakdown.
|
||||
pub num_flops_per_gpu_breakdown: Option<BTreeMap<String, u64>>,
|
||||
/// Optional per-component memory-read breakdown.
|
||||
pub num_read_bytes_per_gpu_breakdown: Option<BTreeMap<String, u64>>,
|
||||
/// Optional per-component memory-write breakdown.
|
||||
pub num_write_bytes_per_gpu_breakdown: Option<BTreeMap<String, u64>>,
|
||||
}
|
||||
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/metrics/perf.py#L58-L63>
|
||||
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct PerfStats {
|
||||
/// Estimated floating point operations per GPU.
|
||||
pub num_flops_per_gpu: u64,
|
||||
/// Estimated bytes read from memory per GPU.
|
||||
pub num_read_bytes_per_gpu: u64,
|
||||
/// Estimated bytes written to memory per GPU.
|
||||
pub num_write_bytes_per_gpu: u64,
|
||||
/// Optional debug-only perf derivation details.
|
||||
pub debug_stats: Option<DebugPerfStats>,
|
||||
}
|
||||
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/compilation/cuda_graph.py#L28-L33>
|
||||
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct CudagraphStat {
|
||||
/// Number of real tokens in the captured batch before padding.
|
||||
pub num_unpadded_tokens: u64,
|
||||
/// Number of padded tokens in the captured batch.
|
||||
pub num_padded_tokens: u64,
|
||||
/// Number of padding positions added for capture/runtime shape alignment.
|
||||
pub num_paddings: u64,
|
||||
/// Runtime mode string associated with this CUDA graph sample.
|
||||
pub runtime_mode: String,
|
||||
}
|
||||
|
||||
/// Stats associated with the scheduler.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/906a8c15d0beded2cca8b357877266c631340f74/vllm/v1/metrics/stats.py#L170-L196>
|
||||
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct SchedulerStats {
|
||||
/// Number of requests in model execution batches.
|
||||
pub num_running_reqs: u64,
|
||||
/// Length of the "waiting" request queue.
|
||||
pub num_waiting_reqs: u64,
|
||||
/// Length of the "skipped waiting" queue.
|
||||
#[serde(default)]
|
||||
pub num_skipped_waiting_reqs: u64,
|
||||
/// Internal DP load-balancing step counter.
|
||||
pub step_counter: u64,
|
||||
/// Internal DP load-balancing wave number.
|
||||
pub current_wave: u64,
|
||||
/// KV-cache usage. `1.0` means 100% usage.
|
||||
pub kv_cache_usage: f64,
|
||||
/// Local prefix cache statistics.
|
||||
pub prefix_cache_stats: PrefixCacheStats,
|
||||
/// External connector prefix cache statistics, when configured.
|
||||
pub connector_prefix_cache_stats: Option<PrefixCacheStats>,
|
||||
/// Sampled KV cache eviction events for residency metrics.
|
||||
pub kv_cache_eviction_events: Vec<KvCacheEvictionEvent>,
|
||||
/// Speculative decoding scheduler stats, when enabled.
|
||||
pub spec_decoding_stats: Option<SpecDecodingStats>,
|
||||
/// Connector-specific KV transfer stats, kept opaque for now.
|
||||
pub kv_connector_stats: Option<BTreeMap<String, OpaqueValue>>,
|
||||
/// Waiting request counts per LoRA adapter.
|
||||
pub waiting_lora_adapters: BTreeMap<String, u64>,
|
||||
/// Running request counts per LoRA adapter.
|
||||
pub running_lora_adapters: BTreeMap<String, u64>,
|
||||
/// CUDA graph runtime stats when graph metrics are enabled.
|
||||
pub cudagraph_stats: Option<CudagraphStat>,
|
||||
/// Estimated MFU/performance stats, when enabled.
|
||||
pub perf_stats: Option<PerfStats>,
|
||||
}
|
||||
@@ -0,0 +1,255 @@
|
||||
use bytemuck::allocation::pod_collect_to_vec;
|
||||
use enum_as_inner::EnumAsInner;
|
||||
use half::{bf16, f16};
|
||||
use rmpv::Value;
|
||||
use serde::{Deserialize, Deserializer, Serialize, Serializer};
|
||||
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
|
||||
|
||||
/// Tensors and ndarrays are encoded with this extension type in Python.
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/v1/serial_utils.py#L41-L43>
|
||||
const CUSTOM_TYPE_RAW_VIEW: i8 = 3;
|
||||
|
||||
#[easy_ext::ext(ShapeExt)]
|
||||
impl [usize] {
|
||||
/// Returned the total number of elements implied by this shape, or `None`
|
||||
/// if the product of the dimensions overflows `usize`.
|
||||
pub fn checked_numel(&self) -> Option<usize> {
|
||||
self.iter().try_fold(1usize, |acc, dim| acc.checked_mul(*dim))
|
||||
}
|
||||
}
|
||||
|
||||
/// Python ndarray/tensor wire tuple encoded as `(dtype, shape, data)`.
|
||||
///
|
||||
/// This matches the custom msgpack representation built by Python
|
||||
/// `serial_utils.encode_ndarray` / `encode_tensor`.
|
||||
///
|
||||
/// Original Python wire encoders:
|
||||
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/v1/serial_utils.py#L237-L273>
|
||||
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/v1/serial_utils.py#L389-L425>
|
||||
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple)]
|
||||
pub struct WireNdArray {
|
||||
pub dtype: String,
|
||||
pub shape: Vec<usize>,
|
||||
pub data: WireArrayData,
|
||||
}
|
||||
|
||||
impl WireNdArray {
|
||||
/// Build a float32 tensor/ndarray backed by native-endian raw-view bytes.
|
||||
pub fn from_f32(shape: Vec<usize>, data: Vec<f32>) -> Result<Self, String> {
|
||||
validate_element_count(&shape, data.len())?;
|
||||
Ok(Self {
|
||||
dtype: "float32".to_string(),
|
||||
shape,
|
||||
data: WireArrayData::RawView(pod_collect_to_vec::<f32, u8>(&data)),
|
||||
})
|
||||
}
|
||||
|
||||
/// Build a float16 tensor/ndarray backed by native-endian raw-view bytes.
|
||||
pub fn from_f16(shape: Vec<usize>, data: Vec<f16>) -> Result<Self, String> {
|
||||
validate_element_count(&shape, data.len())?;
|
||||
Ok(Self {
|
||||
dtype: "float16".to_string(),
|
||||
shape,
|
||||
data: WireArrayData::RawView(pod_collect_to_vec::<f16, u8>(&data)),
|
||||
})
|
||||
}
|
||||
|
||||
/// Build a bfloat16 tensor/ndarray backed by native-endian raw-view bytes.
|
||||
pub fn from_bf16(shape: Vec<usize>, data: Vec<bf16>) -> Result<Self, String> {
|
||||
validate_element_count(&shape, data.len())?;
|
||||
Ok(Self {
|
||||
dtype: "bfloat16".to_string(),
|
||||
shape,
|
||||
data: WireArrayData::RawView(pod_collect_to_vec::<bf16, u8>(&data)),
|
||||
})
|
||||
}
|
||||
|
||||
/// Build an int64 tensor/ndarray backed by native-endian raw-view bytes.
|
||||
pub fn from_i64(shape: Vec<usize>, data: Vec<i64>) -> Result<Self, String> {
|
||||
validate_element_count(&shape, data.len())?;
|
||||
Ok(Self {
|
||||
dtype: "int64".to_string(),
|
||||
shape,
|
||||
data: WireArrayData::RawView(pod_collect_to_vec::<i64, u8>(&data)),
|
||||
})
|
||||
}
|
||||
|
||||
/// Build a uint32 tensor/ndarray backed by native-endian raw-view bytes.
|
||||
pub fn from_u32(shape: Vec<usize>, data: Vec<u32>) -> Result<Self, String> {
|
||||
validate_element_count(&shape, data.len())?;
|
||||
Ok(Self {
|
||||
dtype: "uint32".to_string(),
|
||||
shape,
|
||||
data: WireArrayData::RawView(pod_collect_to_vec::<u32, u8>(&data)),
|
||||
})
|
||||
}
|
||||
|
||||
/// Build a bool tensor/ndarray backed by raw-view bytes.
|
||||
///
|
||||
/// This matches `torch.bool` storage: one byte per element, not a packed
|
||||
/// bitmap. Values are canonicalized as `false -> 0` and `true -> 1`.
|
||||
pub fn from_bool(shape: Vec<usize>, data: Vec<bool>) -> Result<Self, String> {
|
||||
validate_element_count(&shape, data.len())?;
|
||||
Ok(Self {
|
||||
dtype: "bool".to_string(),
|
||||
shape,
|
||||
data: WireArrayData::RawView(data.iter().map(|value| u8::from(*value)).collect()),
|
||||
})
|
||||
}
|
||||
|
||||
/// Build a tensor/ndarray from already-encoded raw-view bytes.
|
||||
///
|
||||
/// Use this as an escape hatch when the caller already owns bytes that
|
||||
/// match the requested `dtype` and `shape`.
|
||||
pub fn from_raw(dtype: impl Into<String>, shape: Vec<usize>, data: Vec<u8>) -> Self {
|
||||
Self {
|
||||
dtype: dtype.into(),
|
||||
shape,
|
||||
data: WireArrayData::RawView(data),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Validate that the number of elements implied by the shape matches the length
|
||||
/// of the data.
|
||||
fn validate_element_count(shape: &[usize], len: usize) -> Result<(), String> {
|
||||
let expected = shape
|
||||
.checked_numel()
|
||||
.ok_or_else(|| format!("tensor shape product overflows usize: {shape:?}"))?;
|
||||
if expected == len {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(format!(
|
||||
"tensor data length {len} does not match shape {shape:?} product {expected}"
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// Python tensor wire tuple encoded as `(dtype, shape, data)`.
|
||||
///
|
||||
/// This is the same wire shape as [`WireNdArray`]; multimodal request payloads
|
||||
/// use it for `torch.Tensor` values.
|
||||
pub type WireTensor = WireNdArray;
|
||||
|
||||
/// Python array/tensor payload reference inside [`WireNdArray`].
|
||||
///
|
||||
/// The data can be either an inline msgpack raw-view extension or an index into
|
||||
/// the multipart aux-frame list carried alongside the primary msgpack frame.
|
||||
///
|
||||
/// Original Python wire encoders:
|
||||
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/v1/serial_utils.py#L237-L273>
|
||||
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/v1/serial_utils.py#L389-L425>
|
||||
#[derive(Debug, Clone, PartialEq, EnumAsInner)]
|
||||
pub enum WireArrayData {
|
||||
/// The index of the aux frame where the raw bytes of this array/tensor are
|
||||
/// stored.
|
||||
AuxIndex(usize),
|
||||
/// The raw bytes of this array/tensor.
|
||||
RawView(Vec<u8>),
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for WireArrayData {
|
||||
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
let value = Value::deserialize(deserializer)?;
|
||||
match value {
|
||||
Value::Ext(tag, bytes) if tag == CUSTOM_TYPE_RAW_VIEW => Ok(Self::RawView(bytes)),
|
||||
Value::Ext(tag, _) => Err(serde::de::Error::custom(format!(
|
||||
"unsupported extension type code {tag}"
|
||||
))),
|
||||
Value::Integer(index) => {
|
||||
index.as_u64().map(|index| Self::AuxIndex(index as usize)).ok_or_else(|| {
|
||||
serde::de::Error::custom("aux frame index must be a non-negative integer")
|
||||
})
|
||||
}
|
||||
other => Err(serde::de::Error::custom(format!(
|
||||
"expected raw-view ext or aux frame index, got {other:?}"
|
||||
))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Serialize for WireArrayData {
|
||||
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
|
||||
where
|
||||
S: Serializer,
|
||||
{
|
||||
// TODO: outbound request serialization currently only supports inline
|
||||
// raw-view bytes. Emitting aux frames needs transport-level plumbing;
|
||||
// serializing `AuxIndex` here only preserves an already-built reference.
|
||||
match self {
|
||||
Self::AuxIndex(index) => serializer.serialize_u64(*index as u64),
|
||||
Self::RawView(bytes) => {
|
||||
Value::Ext(CUSTOM_TYPE_RAW_VIEW, bytes.clone()).serialize(serializer)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn constructors_build_raw_view_tensors() {
|
||||
let f32_tensor = WireNdArray::from_f32(vec![2], vec![1.0, 2.5]).unwrap();
|
||||
assert_eq!(f32_tensor.dtype, "float32");
|
||||
assert_eq!(f32_tensor.shape, vec![2]);
|
||||
assert_eq!(
|
||||
f32_tensor.data.into_raw_view().expect("raw view"),
|
||||
[1.0_f32, 2.5].into_iter().flat_map(f32::to_ne_bytes).collect::<Vec<_>>()
|
||||
);
|
||||
|
||||
let f16_tensor =
|
||||
WireNdArray::from_f16(vec![2], vec![f16::from_f32(1.0), f16::from_f32(2.5)]).unwrap();
|
||||
assert_eq!(f16_tensor.dtype, "float16");
|
||||
assert_eq!(f16_tensor.shape, vec![2]);
|
||||
assert_eq!(f16_tensor.data.into_raw_view().expect("raw view").len(), 4);
|
||||
|
||||
let bf16_tensor =
|
||||
WireNdArray::from_bf16(vec![2], vec![bf16::from_f32(1.0), bf16::from_f32(2.5)])
|
||||
.unwrap();
|
||||
assert_eq!(bf16_tensor.dtype, "bfloat16");
|
||||
assert_eq!(bf16_tensor.shape, vec![2]);
|
||||
assert_eq!(bf16_tensor.data.into_raw_view().expect("raw view").len(), 4);
|
||||
|
||||
let i64_tensor = WireNdArray::from_i64(vec![1], vec![-7]).unwrap();
|
||||
assert_eq!(i64_tensor.dtype, "int64");
|
||||
assert_eq!(
|
||||
i64_tensor.data.into_raw_view().expect("raw view"),
|
||||
(-7_i64).to_ne_bytes()
|
||||
);
|
||||
|
||||
let u32_tensor = WireNdArray::from_u32(vec![1], vec![42]).unwrap();
|
||||
assert_eq!(u32_tensor.dtype, "uint32");
|
||||
assert_eq!(
|
||||
u32_tensor.data.into_raw_view().expect("raw view"),
|
||||
42_u32.to_ne_bytes()
|
||||
);
|
||||
|
||||
let bool_tensor = WireNdArray::from_bool(vec![2], vec![false, true]).unwrap();
|
||||
assert_eq!(bool_tensor.dtype, "bool");
|
||||
assert_eq!(
|
||||
bool_tensor.data.into_raw_view().expect("raw view"),
|
||||
vec![0, 1]
|
||||
);
|
||||
|
||||
let raw_tensor = WireNdArray::from_raw("custom", vec![3], vec![1, 2, 3]);
|
||||
assert_eq!(raw_tensor.dtype, "custom");
|
||||
assert_eq!(raw_tensor.shape, vec![3]);
|
||||
assert_eq!(
|
||||
raw_tensor.data.into_raw_view().expect("raw view"),
|
||||
vec![1, 2, 3]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn constructors_validate_shape_product() {
|
||||
let err = WireNdArray::from_f32(vec![2, 2], vec![1.0, 2.0]).unwrap_err();
|
||||
assert!(err.contains("does not match shape"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,295 @@
|
||||
use std::future::Future;
|
||||
use std::path::Path;
|
||||
use std::pin::Pin;
|
||||
use std::time::Duration;
|
||||
|
||||
use tempfile::TempDir;
|
||||
use tokio::sync::oneshot;
|
||||
use zeromq::prelude::{Socket, SocketRecv, SocketSend};
|
||||
use zeromq::util::PeerIdentity;
|
||||
use zeromq::{DealerSocket, PushSocket, SocketOptions, SubSocket, ZmqMessage};
|
||||
|
||||
use crate::EngineId;
|
||||
use crate::protocol::ModelDtype;
|
||||
use crate::protocol::handshake::{EngineCoreReadyResponse, HandshakeInitMessage, ReadyMessage};
|
||||
|
||||
/// Per-test IPC endpoint namespace backed by a unique temporary directory.
|
||||
///
|
||||
/// Using one directory per test avoids endpoint collisions without requiring
|
||||
/// ad-hoc unique-name generation at each call site.
|
||||
#[derive(Debug)]
|
||||
pub struct IpcNamespace {
|
||||
dir: TempDir,
|
||||
}
|
||||
|
||||
impl IpcNamespace {
|
||||
/// Create a fresh namespace for one test case.
|
||||
pub fn new() -> std::io::Result<Self> {
|
||||
Ok(Self {
|
||||
dir: TempDir::new()?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Build one `ipc://...` endpoint under this namespace.
|
||||
pub fn endpoint(&self, name: impl AsRef<Path>) -> String {
|
||||
let path = self.dir.path().join(name);
|
||||
format!("ipc://{}", path.to_string_lossy())
|
||||
}
|
||||
|
||||
/// Endpoint used for the initial READY/HELLO handshake.
|
||||
pub fn handshake_endpoint(&self) -> String {
|
||||
self.endpoint("handshake.sock")
|
||||
}
|
||||
|
||||
/// Endpoint used for engine-core request traffic.
|
||||
pub fn input_endpoint(&self) -> String {
|
||||
self.endpoint("input.sock")
|
||||
}
|
||||
|
||||
/// Endpoint used for engine-core output traffic.
|
||||
pub fn output_endpoint(&self) -> String {
|
||||
self.endpoint("output.sock")
|
||||
}
|
||||
}
|
||||
|
||||
/// Construct a standard local READY message used by mock engines in tests.
|
||||
fn ready_message(status: &str) -> ReadyMessage {
|
||||
ReadyMessage {
|
||||
status: Some(status.to_string()),
|
||||
local: Some(true),
|
||||
headless: Some(true),
|
||||
parallel_config_hash: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Construct a default ready response payload for mock engine input
|
||||
/// registration.
|
||||
fn ready_response_payload() -> Vec<u8> {
|
||||
rmp_serde::to_vec_named(&EngineCoreReadyResponse {
|
||||
max_model_len: 4096,
|
||||
num_gpu_blocks: 0,
|
||||
dp_stats_address: None,
|
||||
dtype: Some(ModelDtype::Float32),
|
||||
})
|
||||
.expect("encode ready response payload")
|
||||
}
|
||||
|
||||
/// Coordinator-side sockets connected by one mock engine when coordinator mode
|
||||
/// is enabled.
|
||||
pub struct MockCoordinatorConnections {
|
||||
/// Subscription socket that receives coordinator broadcasts such as
|
||||
/// `START_DP_WAVE`.
|
||||
pub input_sub: SubSocket,
|
||||
/// Push socket used to send coordinator-only `EngineCoreOutputs` back to
|
||||
/// the frontend.
|
||||
pub output_push: PushSocket,
|
||||
}
|
||||
|
||||
/// Fully connected mock engine transport state used by tests.
|
||||
pub struct MockEngineConnections {
|
||||
/// Decoded INIT message sent by the frontend during handshake.
|
||||
pub init: HandshakeInitMessage,
|
||||
/// Socket used to receive frontend requests.
|
||||
pub dealer: DealerSocket,
|
||||
/// Socket used to publish normal request outputs back to the frontend.
|
||||
pub push: PushSocket,
|
||||
/// Optional coordinator sockets when the client enabled the in-process
|
||||
/// coordinator.
|
||||
pub coordinator: Option<MockCoordinatorConnections>,
|
||||
}
|
||||
|
||||
/// Complete the engine-core handshake and connect mock input/output sockets
|
||||
/// plus optional coordinator sockets.
|
||||
pub async fn setup_mock_engine_connections(
|
||||
engine_handshake: String,
|
||||
engine_id: impl Into<EngineId>,
|
||||
) -> MockEngineConnections {
|
||||
// Wait for the client to bind the handshake socket before connecting.
|
||||
// A fixed sleep is racy under CI load; instead poll for the socket file.
|
||||
let socket_path = engine_handshake
|
||||
.strip_prefix("ipc://")
|
||||
.expect("handshake address must be ipc://");
|
||||
for _ in 0..100 {
|
||||
if Path::new(socket_path).exists() {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
|
||||
let peer_identity = PeerIdentity::try_from(engine_id.into()).expect("peer id");
|
||||
|
||||
let mut options = SocketOptions::default();
|
||||
options.peer_identity(peer_identity.clone());
|
||||
let mut handshake = DealerSocket::with_options(options);
|
||||
handshake
|
||||
.connect(&engine_handshake)
|
||||
.await
|
||||
.expect("connect mock engine handshake socket");
|
||||
handshake
|
||||
.send(ZmqMessage::from(
|
||||
rmp_serde::to_vec_named(&ready_message("HELLO")).expect("encode HELLO ready message"),
|
||||
))
|
||||
.await
|
||||
.expect("send HELLO ready message");
|
||||
|
||||
let init_frames = handshake.recv().await.expect("receive handshake init message").into_vec();
|
||||
assert_eq!(init_frames.len(), 1);
|
||||
let init: HandshakeInitMessage =
|
||||
rmp_serde::from_slice(init_frames[0].as_ref()).expect("decode handshake init message");
|
||||
|
||||
let mut input_options = SocketOptions::default();
|
||||
input_options.peer_identity(peer_identity);
|
||||
let mut dealer = DealerSocket::with_options(input_options);
|
||||
dealer
|
||||
.connect(&init.addresses.inputs[0])
|
||||
.await
|
||||
.expect("connect mock engine input socket");
|
||||
dealer
|
||||
.send(ZmqMessage::from(ready_response_payload()))
|
||||
.await
|
||||
.expect("send mock engine input ready frame");
|
||||
|
||||
let mut push = PushSocket::new();
|
||||
push.connect(&init.addresses.outputs[0])
|
||||
.await
|
||||
.expect("connect mock engine output socket");
|
||||
|
||||
let coordinator = match (
|
||||
init.addresses.coordinator_input.as_deref(),
|
||||
init.addresses.coordinator_output.as_deref(),
|
||||
) {
|
||||
(Some(coordinator_input), Some(coordinator_output)) => {
|
||||
let mut input_sub = SubSocket::new();
|
||||
input_sub
|
||||
.connect(coordinator_input)
|
||||
.await
|
||||
.expect("connect mock engine coordinator input socket");
|
||||
input_sub
|
||||
.subscribe("")
|
||||
.await
|
||||
.expect("subscribe mock engine coordinator input socket");
|
||||
|
||||
let mut output_push = PushSocket::new();
|
||||
output_push
|
||||
.connect(coordinator_output)
|
||||
.await
|
||||
.expect("connect mock engine coordinator output socket");
|
||||
|
||||
let ready =
|
||||
input_sub.recv().await.expect("receive coordinator READY marker").into_vec();
|
||||
assert_eq!(ready.len(), 1);
|
||||
assert_eq!(ready[0].as_ref(), b"READY");
|
||||
|
||||
Some(MockCoordinatorConnections {
|
||||
input_sub,
|
||||
output_push,
|
||||
})
|
||||
}
|
||||
(None, None) => None,
|
||||
_ => panic!("coordinator handshake addresses must be both present or both absent"),
|
||||
};
|
||||
|
||||
handshake
|
||||
.send(ZmqMessage::from(
|
||||
rmp_serde::to_vec_named(&ready_message("READY")).expect("encode READY ready message"),
|
||||
))
|
||||
.await
|
||||
.expect("send READY ready message");
|
||||
|
||||
MockEngineConnections {
|
||||
init,
|
||||
dealer,
|
||||
push,
|
||||
coordinator,
|
||||
}
|
||||
}
|
||||
|
||||
/// Connect one mock engine directly to already-bootstrapped frontend
|
||||
/// input/output sockets.
|
||||
pub async fn setup_bootstrapped_mock_engine(
|
||||
input_address: String,
|
||||
output_address: String,
|
||||
engine_id: impl Into<EngineId>,
|
||||
) -> (DealerSocket, PushSocket) {
|
||||
for endpoint in [&input_address, &output_address] {
|
||||
if let Some(socket_path) = endpoint.strip_prefix("ipc://") {
|
||||
for _ in 0..100 {
|
||||
if Path::new(socket_path).exists() {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let peer_identity = PeerIdentity::try_from(engine_id.into()).expect("peer id");
|
||||
let mut input_options = SocketOptions::default();
|
||||
input_options.peer_identity(peer_identity);
|
||||
let mut dealer = DealerSocket::with_options(input_options);
|
||||
dealer.connect(&input_address).await.expect("connect mock engine input socket");
|
||||
dealer
|
||||
.send(ZmqMessage::from(ready_response_payload()))
|
||||
.await
|
||||
.expect("send mock engine input ready frame");
|
||||
|
||||
let mut push = PushSocket::new();
|
||||
push.connect(&output_address).await.expect("connect mock engine output socket");
|
||||
|
||||
(dealer, push)
|
||||
}
|
||||
|
||||
/// Complete the engine-core handshake and connect mock input/output sockets.
|
||||
///
|
||||
/// This returns the decoded handshake init message plus the `DealerSocket` used
|
||||
/// to receive client requests and the `PushSocket` used to send engine outputs
|
||||
/// back to the client.
|
||||
pub async fn setup_mock_engine_with_init(
|
||||
engine_handshake: String,
|
||||
engine_id: impl Into<EngineId>,
|
||||
) -> (HandshakeInitMessage, DealerSocket, PushSocket) {
|
||||
let MockEngineConnections {
|
||||
init, dealer, push, ..
|
||||
} = setup_mock_engine_connections(engine_handshake, engine_id).await;
|
||||
(init, dealer, push)
|
||||
}
|
||||
|
||||
/// Complete the engine-core handshake and connect mock input/output sockets.
|
||||
///
|
||||
/// This returns the `DealerSocket` used to receive client requests and the
|
||||
/// `PushSocket` used to send engine outputs back to the client.
|
||||
pub async fn setup_mock_engine(
|
||||
engine_handshake: String,
|
||||
engine_id: impl Into<EngineId>,
|
||||
) -> (DealerSocket, PushSocket) {
|
||||
let (_, dealer, push) = setup_mock_engine_with_init(engine_handshake, engine_id).await;
|
||||
(dealer, push)
|
||||
}
|
||||
|
||||
/// Spawn a mock engine task and keep its sockets alive until the returned
|
||||
/// shutdown sender is triggered by the test.
|
||||
///
|
||||
/// The script borrows the connected sockets mutably while it runs. After the
|
||||
/// script completes, this helper keeps the sockets alive until the test
|
||||
/// explicitly signals shutdown.
|
||||
pub fn spawn_mock_engine_task<F>(
|
||||
engine_handshake: String,
|
||||
engine_id: impl Into<EngineId>,
|
||||
run: F,
|
||||
) -> (oneshot::Sender<()>, tokio::task::JoinHandle<()>)
|
||||
where
|
||||
F: for<'a> FnOnce(
|
||||
&'a mut DealerSocket,
|
||||
&'a mut PushSocket,
|
||||
) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>>
|
||||
+ Send
|
||||
+ 'static,
|
||||
{
|
||||
let (shutdown_tx, shutdown_rx) = oneshot::channel();
|
||||
let engine_id = engine_id.into();
|
||||
let engine_task = tokio::spawn(async move {
|
||||
let (mut dealer, mut push) = setup_mock_engine(engine_handshake, engine_id).await;
|
||||
run(&mut dealer, &mut push).await;
|
||||
let _ = shutdown_rx.await;
|
||||
});
|
||||
(shutdown_tx, engine_task)
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1 @@
|
||||
mod client;
|
||||
+356
@@ -0,0 +1,356 @@
|
||||
#!/usr/bin/env -S uv run
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# /// script
|
||||
# requires-python = ">=3.11"
|
||||
# dependencies = [
|
||||
# "msgspec>=0.19,<1",
|
||||
# "msgpack>=1,<2",
|
||||
# "numpy>=2,<3",
|
||||
# ]
|
||||
# ///
|
||||
|
||||
from enum import Enum, IntEnum
|
||||
|
||||
import msgpack
|
||||
import msgspec
|
||||
import numpy as np
|
||||
|
||||
|
||||
class RequestOutputKind(Enum):
|
||||
DELTA = 1
|
||||
FINAL_ONLY = 2
|
||||
|
||||
|
||||
class FinishReason(IntEnum):
|
||||
STOP = 0
|
||||
LENGTH = 1
|
||||
ABORT = 2
|
||||
ERROR = 3
|
||||
REPETITION = 4
|
||||
|
||||
|
||||
class EngineCoreSamplingParams(msgspec.Struct, dict=True):
|
||||
temperature: float = 1.0
|
||||
top_p: float = 1.0
|
||||
top_k: int = 0
|
||||
seed: int | None = None
|
||||
max_tokens: int = 65536
|
||||
min_tokens: int = 0
|
||||
min_p: float = 0.0
|
||||
frequency_penalty: float = 0.0
|
||||
presence_penalty: float = 0.0
|
||||
repetition_penalty: float = 1.0
|
||||
stop_token_ids: list[int] = []
|
||||
_eos_token_id: int | None = None
|
||||
_all_stop_token_ids: set[int] = set()
|
||||
output_kind: RequestOutputKind = RequestOutputKind.DELTA
|
||||
|
||||
|
||||
class EngineCoreRequest(
|
||||
msgspec.Struct,
|
||||
array_like=True,
|
||||
omit_defaults=True,
|
||||
):
|
||||
request_id: str
|
||||
prompt_token_ids: list[int] | None
|
||||
mm_features: object | None
|
||||
sampling_params: EngineCoreSamplingParams | None
|
||||
pooling_params: object | None
|
||||
arrival_time: float
|
||||
lora_request: object | None = None
|
||||
cache_salt: str | None = None
|
||||
data_parallel_rank: int | None = None
|
||||
prompt_embeds: object | None = None
|
||||
prompt_is_token_ids: list[bool] | None = None
|
||||
client_index: int = 0
|
||||
current_wave: int = 0
|
||||
priority: int = 0
|
||||
trace_headers: dict[str, str] | None = None
|
||||
resumable: bool = False
|
||||
external_req_id: str | None = None
|
||||
reasoning_ended: bool | None = None
|
||||
reasoning_parser_kwargs: dict[str, object] | None = None
|
||||
abort_immediately: bool = False
|
||||
|
||||
|
||||
class EngineCoreOutput(
|
||||
msgspec.Struct,
|
||||
array_like=True,
|
||||
omit_defaults=True,
|
||||
):
|
||||
request_id: str
|
||||
new_token_ids: list[int]
|
||||
new_logprobs: object | None = None
|
||||
new_prompt_logprobs_tensors: object | None = None
|
||||
pooling_output: object | None = None
|
||||
finish_reason: FinishReason | None = None
|
||||
stop_reason: int | str | None = None
|
||||
events: object | None = None
|
||||
kv_transfer_params: object | None = None
|
||||
trace_headers: object | None = None
|
||||
prefill_stats: object | None = None
|
||||
routed_experts: object | None = None
|
||||
num_nans_in_logits: int = 0
|
||||
|
||||
|
||||
class EngineCoreOutputs(
|
||||
msgspec.Struct,
|
||||
array_like=True,
|
||||
omit_defaults=True,
|
||||
):
|
||||
engine_index: int = 0
|
||||
outputs: list[EngineCoreOutput] = []
|
||||
scheduler_stats: object | None = None
|
||||
timestamp: float = 0.0
|
||||
utility_output: object | None = None
|
||||
finished_requests: set[str] | None = None
|
||||
wave_complete: int | None = None
|
||||
start_wave: int | None = None
|
||||
|
||||
|
||||
request = EngineCoreRequest(
|
||||
request_id="req-1",
|
||||
prompt_token_ids=[11, 22],
|
||||
mm_features=None,
|
||||
sampling_params=EngineCoreSamplingParams(
|
||||
temperature=0.8,
|
||||
top_p=0.9,
|
||||
top_k=8,
|
||||
seed=None,
|
||||
max_tokens=32,
|
||||
min_tokens=1,
|
||||
min_p=0.0,
|
||||
frequency_penalty=0.0,
|
||||
presence_penalty=0.0,
|
||||
repetition_penalty=1.0,
|
||||
stop_token_ids=[151643],
|
||||
_eos_token_id=151645,
|
||||
_all_stop_token_ids={151643, 151645},
|
||||
output_kind=RequestOutputKind.FINAL_ONLY,
|
||||
),
|
||||
pooling_params=None,
|
||||
arrival_time=42.5,
|
||||
client_index=0,
|
||||
)
|
||||
|
||||
multimodal_tensor = np.array([[1.0, 2.0], [3.5, 4.25]], dtype=np.float32)
|
||||
multimodal_features = [
|
||||
{
|
||||
"data": {
|
||||
"pixel_values": {
|
||||
"data": [
|
||||
"float32",
|
||||
[2, 2],
|
||||
msgpack.ExtType(3, multimodal_tensor.tobytes()),
|
||||
],
|
||||
"field": [
|
||||
"flat",
|
||||
{
|
||||
"slices": [[0, 2, None]],
|
||||
"dim": 0,
|
||||
"keep_on_cpu": False,
|
||||
},
|
||||
],
|
||||
}
|
||||
},
|
||||
"modality": "image",
|
||||
"identifier": "mm-cache-key",
|
||||
"mm_position": {
|
||||
"offset": 1,
|
||||
"length": 2,
|
||||
"is_embed": None,
|
||||
},
|
||||
"mm_hash": "processor-hash",
|
||||
}
|
||||
]
|
||||
multimodal_request_wire = [
|
||||
"req-mm",
|
||||
[101, 102, 103, 104],
|
||||
multimodal_features,
|
||||
None,
|
||||
None,
|
||||
43.5,
|
||||
]
|
||||
|
||||
outputs = EngineCoreOutputs(
|
||||
outputs=[
|
||||
EngineCoreOutput(
|
||||
request_id="req-1",
|
||||
new_token_ids=[7, 8],
|
||||
finish_reason=FinishReason.LENGTH,
|
||||
)
|
||||
],
|
||||
finished_requests={"req-1"},
|
||||
)
|
||||
|
||||
|
||||
def encode_ndarray(
|
||||
array: np.ndarray,
|
||||
buffers: list[bytes],
|
||||
*,
|
||||
size_threshold: int = 256,
|
||||
):
|
||||
arr_data = array.data if array.flags.c_contiguous else array.tobytes()
|
||||
if not array.shape or array.nbytes < size_threshold:
|
||||
data = msgpack.ExtType(3, bytes(arr_data))
|
||||
else:
|
||||
data = len(buffers)
|
||||
buffers.append(bytes(arr_data))
|
||||
return [array.dtype.str, list(array.shape), data]
|
||||
|
||||
|
||||
def encode_tensor_like(
|
||||
dtype: str,
|
||||
shape: list[int],
|
||||
payload: bytes,
|
||||
buffers: list[bytes],
|
||||
*,
|
||||
size_threshold: int = 256,
|
||||
):
|
||||
if len(payload) < size_threshold:
|
||||
data = msgpack.ExtType(3, payload)
|
||||
else:
|
||||
data = len(buffers)
|
||||
buffers.append(payload)
|
||||
return [dtype, shape, data]
|
||||
|
||||
|
||||
def encode_output_frames(obj, *, size_threshold: int = 256) -> list[bytes]:
|
||||
buffers = [b""]
|
||||
|
||||
def transform(value):
|
||||
if isinstance(value, np.ndarray):
|
||||
return encode_ndarray(value, buffers, size_threshold=size_threshold)
|
||||
if (
|
||||
isinstance(value, tuple)
|
||||
and len(value) == 3
|
||||
and value[0] in ("int32", "int64", "float32")
|
||||
):
|
||||
dtype, shape, payload = value
|
||||
return encode_tensor_like(
|
||||
dtype,
|
||||
shape,
|
||||
payload,
|
||||
buffers,
|
||||
size_threshold=size_threshold,
|
||||
)
|
||||
if type(value) is list:
|
||||
return [transform(v) for v in value]
|
||||
if type(value) is tuple:
|
||||
return [transform(v) for v in value]
|
||||
if type(value) is dict:
|
||||
return {k: transform(v) for k, v in value.items()}
|
||||
return value
|
||||
|
||||
buffers[0] = msgpack.packb(transform(obj), use_bin_type=True)
|
||||
return buffers
|
||||
|
||||
|
||||
def engine_output_wire(
|
||||
request_id: str,
|
||||
*,
|
||||
new_logprobs=None,
|
||||
new_prompt_logprobs_tensors=None,
|
||||
):
|
||||
return [
|
||||
request_id,
|
||||
[7, 8],
|
||||
new_logprobs,
|
||||
new_prompt_logprobs_tensors,
|
||||
None,
|
||||
int(FinishReason.LENGTH),
|
||||
]
|
||||
|
||||
|
||||
def engine_outputs_wire(output):
|
||||
return [0, [output], None, 0.0, None, ["req-1"]]
|
||||
|
||||
|
||||
inline_logprobs = engine_outputs_wire(
|
||||
engine_output_wire(
|
||||
"req-1",
|
||||
new_logprobs=(
|
||||
np.array([[1, 2, 3], [4, 5, 6]], dtype=np.int64),
|
||||
np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float32),
|
||||
np.array([1, 2], dtype=np.int64),
|
||||
None,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
multipart_logprobs = engine_outputs_wire(
|
||||
engine_output_wire(
|
||||
"req-1",
|
||||
new_logprobs=(
|
||||
np.array([[1, 2, 3], [4, 5, 6]], dtype=np.int64),
|
||||
np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float32),
|
||||
np.array([1, 2], dtype=np.int64),
|
||||
None,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
inline_prompt_logprobs = engine_outputs_wire(
|
||||
engine_output_wire(
|
||||
"req-1",
|
||||
new_prompt_logprobs_tensors=(
|
||||
(
|
||||
"int64",
|
||||
[2, 3],
|
||||
np.array([[10, 11, 12], [13, 14, 15]], dtype=np.int64).tobytes(),
|
||||
),
|
||||
(
|
||||
"float32",
|
||||
[2, 3],
|
||||
np.array(
|
||||
[[10, 11, 12], [13, 14, 15]],
|
||||
dtype=np.float32,
|
||||
).tobytes(),
|
||||
),
|
||||
("int64", [2], np.array([3, 4], dtype=np.int64).tobytes()),
|
||||
None,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
multipart_prompt_logprobs = engine_outputs_wire(
|
||||
engine_output_wire(
|
||||
"req-1",
|
||||
new_prompt_logprobs_tensors=(
|
||||
(
|
||||
"int64",
|
||||
[2, 3],
|
||||
np.array([[10, 11, 12], [13, 14, 15]], dtype=np.int64).tobytes(),
|
||||
),
|
||||
(
|
||||
"float32",
|
||||
[2, 3],
|
||||
np.array(
|
||||
[[10, 11, 12], [13, 14, 15]],
|
||||
dtype=np.float32,
|
||||
).tobytes(),
|
||||
),
|
||||
("int64", [2], np.array([3, 4], dtype=np.int64).tobytes()),
|
||||
None,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
print(msgspec.msgpack.encode(request).hex())
|
||||
print(msgpack.packb(multimodal_request_wire, use_bin_type=True).hex())
|
||||
print(msgspec.msgpack.encode(outputs).hex())
|
||||
print(" ".join(frame.hex() for frame in encode_output_frames(inline_logprobs)))
|
||||
print(
|
||||
" ".join(
|
||||
frame.hex()
|
||||
for frame in encode_output_frames(multipart_logprobs, size_threshold=1)
|
||||
)
|
||||
)
|
||||
print(" ".join(frame.hex() for frame in encode_output_frames(inline_prompt_logprobs)))
|
||||
print(
|
||||
" ".join(
|
||||
frame.hex()
|
||||
for frame in encode_output_frames(multipart_prompt_logprobs, size_threshold=1)
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,606 @@
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::fmt::Debug;
|
||||
use std::ops::Deref;
|
||||
use std::time::Duration;
|
||||
|
||||
use bytes::Bytes;
|
||||
use enum_as_inner::EnumAsInner;
|
||||
use thiserror_ext::AsReport;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::time::timeout;
|
||||
use tracing::{debug, error, info, trace, warn};
|
||||
use zeromq::prelude::{Socket, SocketRecv, SocketSend};
|
||||
use zeromq::util::PeerIdentity;
|
||||
use zeromq::{PullSocket, RouterSendHalf, RouterSocket, ZmqError, ZmqMessage};
|
||||
|
||||
use crate::coordinator::CoordinatorBootstrap;
|
||||
use crate::error::{Error, Result, bail_unexpected_handshake_message};
|
||||
use crate::protocol::handshake::{
|
||||
EngineCoreReadyResponse, HandshakeAddresses, HandshakeInitMessage, ReadyMessage,
|
||||
};
|
||||
use crate::protocol::{
|
||||
EngineCoreOutputs, decode_engine_core_outputs, decode_msgpack, encode_msgpack,
|
||||
};
|
||||
|
||||
/// Dedicated single-frame sentinel emitted by Python `EngineCoreProc` when the
|
||||
/// engine dies.
|
||||
pub const ENGINE_CORE_DEAD_SENTINEL: &[u8] = b"ENGINE_CORE_DEAD";
|
||||
|
||||
/// Opaque routing identity of one engine on the frontend transport.
|
||||
#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub struct EngineId(Bytes);
|
||||
|
||||
impl Debug for EngineId {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
// Display the engine id as a hex string for easier debugging.
|
||||
write!(f, "EngineId({})", hex::encode(&self.0))
|
||||
}
|
||||
}
|
||||
|
||||
impl EngineId {
|
||||
/// Convert the engine id into a ZMQ frame for sending.
|
||||
pub fn to_frame(&self) -> Bytes {
|
||||
self.0.clone()
|
||||
}
|
||||
|
||||
/// Convert the engine id into a ZMQ frame for sending.
|
||||
pub fn into_frame(self) -> Bytes {
|
||||
self.0
|
||||
}
|
||||
|
||||
/// Parse the Python-compatible engine index encoded in the routing
|
||||
/// identity.
|
||||
///
|
||||
/// Python `EngineCoreProc` currently uses a two-byte little-endian engine
|
||||
/// index as its ROUTER/DEALER identity. Coordinator control messages
|
||||
/// such as `START_DP_WAVE(exclude_engine_index)` need that engine-side
|
||||
/// index rather than any frontend-local ordering.
|
||||
pub fn engine_index(&self) -> Option<u32> {
|
||||
if self.len() != 2 {
|
||||
return None;
|
||||
}
|
||||
Some(u16::from_le_bytes([self[0], self[1]]) as u32)
|
||||
}
|
||||
|
||||
/// Construct an engine id from the Python-compatible engine index encoding
|
||||
/// (two-byte little-endian).
|
||||
pub fn from_engine_index(value: u32) -> Self {
|
||||
Self(Bytes::copy_from_slice(&(value as u16).to_le_bytes()))
|
||||
}
|
||||
}
|
||||
|
||||
impl Deref for EngineId {
|
||||
type Target = [u8];
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
self.0.as_ref()
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Vec<u8>> for EngineId {
|
||||
fn from(value: Vec<u8>) -> Self {
|
||||
Self(Bytes::from(value))
|
||||
}
|
||||
}
|
||||
|
||||
impl<const N: usize> From<&[u8; N]> for EngineId {
|
||||
fn from(value: &[u8; N]) -> Self {
|
||||
Self(Bytes::copy_from_slice(value))
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<EngineId> for PeerIdentity {
|
||||
type Error = ZmqError;
|
||||
|
||||
fn try_from(value: EngineId) -> std::result::Result<Self, Self::Error> {
|
||||
PeerIdentity::try_from(value.into_frame())
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-engine handshake result collected while bootstrapping one shared
|
||||
/// transport.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ConnectedEngine {
|
||||
/// The identity of the connected engine.
|
||||
pub engine_id: EngineId,
|
||||
/// Post-initialization configuration received from the engine on the input
|
||||
/// socket registration message. `None` until the registration is received.
|
||||
pub ready_response: Option<EngineCoreReadyResponse>,
|
||||
}
|
||||
|
||||
/// Represents the connected shared transport plus all registered engines after
|
||||
/// a successful multi-engine startup handshake.
|
||||
pub struct ConnectedTransport {
|
||||
/// The local address of the shared input socket that all engines connect to
|
||||
/// for receiving requests.
|
||||
pub input_address: String,
|
||||
/// The local address of the shared output socket that all engines connect
|
||||
/// to for sending responses.
|
||||
pub output_address: String,
|
||||
/// All engines connected through the startup handshake.
|
||||
pub engines: Vec<ConnectedEngine>,
|
||||
/// Optional engine-facing coordinator transport used for in-process wave
|
||||
/// coordination.
|
||||
pub coordinator: Option<CoordinatorBootstrap>,
|
||||
|
||||
/// The sending half of the shared input socket.
|
||||
pub input_send: RouterSendHalf,
|
||||
/// The shared output socket for receiving responses from all engines.
|
||||
pub output_socket: PullSocket,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, EnumAsInner)]
|
||||
enum EngineStartupState {
|
||||
HelloReceived,
|
||||
ReadyReceived,
|
||||
}
|
||||
|
||||
/// Connect to one or more engines through the startup handshake protocol,
|
||||
/// returning the shared data-plane transport plus the registered engines.
|
||||
pub async fn connect_handshake(
|
||||
handshake_address: &str,
|
||||
engine_count: usize,
|
||||
local_host: &str,
|
||||
local_input_address: Option<&str>,
|
||||
local_output_address: Option<&str>,
|
||||
enable_inproc_coordinator: bool,
|
||||
ready_timeout: Duration,
|
||||
) -> Result<ConnectedTransport> {
|
||||
if engine_count == 0 {
|
||||
bail_unexpected_handshake_message!("expected engine_count >= 1");
|
||||
}
|
||||
|
||||
info!(
|
||||
engine_count,
|
||||
handshake_address, "waiting for engines to connect"
|
||||
);
|
||||
|
||||
// 1. Bind shared local input/output sockets first so every engine receives the same data-plane
|
||||
// addresses during handshake.
|
||||
debug!(
|
||||
local_host,
|
||||
?ready_timeout,
|
||||
engine_count,
|
||||
"binding shared transport sockets"
|
||||
);
|
||||
let (input_address, mut input_socket, output_address, output_socket) =
|
||||
bind_local_sockets(local_host, local_input_address, local_output_address).await?;
|
||||
info!(%input_address, %output_address, "bound local transport sockets");
|
||||
|
||||
let mut coordinator = if enable_inproc_coordinator {
|
||||
Some(CoordinatorBootstrap::bind(local_host).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// 2. Bind the shared handshake socket once. All engines connect to this socket with their own
|
||||
// identities, and startup order does not matter.
|
||||
let mut handshake_socket = RouterSocket::new();
|
||||
handshake_socket.bind(handshake_address).await?;
|
||||
|
||||
let mut engines = BTreeMap::new();
|
||||
|
||||
// 3. Receive HELLO from every engine and send a matching INIT. When coordinator mode is
|
||||
// enabled, the engines will not emit READY until the coordinator barrier below completes.
|
||||
while engines.len() < engine_count {
|
||||
debug!(
|
||||
handshake_address,
|
||||
connected = engines.len(),
|
||||
waiting_for = engine_count,
|
||||
"waiting for engine HELLO"
|
||||
);
|
||||
let message = timeout(ready_timeout, handshake_socket.recv()).await.map_err(|_| {
|
||||
Error::HandshakeTimeout {
|
||||
stage: "HELLO",
|
||||
timeout: ready_timeout,
|
||||
}
|
||||
})??;
|
||||
let (engine_id, handshake_message) = decode_handshake_message(message, None)?;
|
||||
match handshake_message.status.as_deref() {
|
||||
Some("HELLO") => {
|
||||
if engines.contains_key(&engine_id) {
|
||||
bail_unexpected_handshake_message!(
|
||||
"duplicate engine id {engine_id:?} observed during startup handshake"
|
||||
);
|
||||
}
|
||||
debug!(handshake_address, ?engine_id, "received HELLO from engine");
|
||||
|
||||
send_init_message(
|
||||
&mut handshake_socket,
|
||||
&engine_id,
|
||||
&input_address,
|
||||
&output_address,
|
||||
coordinator.as_ref(),
|
||||
)
|
||||
.await?;
|
||||
debug!(handshake_address, ?engine_id, "sent INIT to engine");
|
||||
|
||||
engines.insert(engine_id.clone(), EngineStartupState::HelloReceived);
|
||||
}
|
||||
Some("READY") => {
|
||||
if coordinator.is_some() {
|
||||
bail_unexpected_handshake_message!(
|
||||
"received READY for engine id {engine_id:?} before coordinator startup gate completed"
|
||||
);
|
||||
}
|
||||
let state = match engines.get_mut(&engine_id) {
|
||||
Some(state) if !state.is_ready_received() => state,
|
||||
_ => {
|
||||
bail_unexpected_handshake_message!(
|
||||
"received READY for unexpected or duplicate engine id {engine_id:?}"
|
||||
);
|
||||
}
|
||||
};
|
||||
debug!(
|
||||
handshake_address,
|
||||
?engine_id,
|
||||
?handshake_message,
|
||||
"received overlapping READY from engine during HELLO phase"
|
||||
);
|
||||
*state = EngineStartupState::ReadyReceived;
|
||||
}
|
||||
other => {
|
||||
bail_unexpected_handshake_message!("unexpected handshake status {other:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Optional coordinator startup gate. Without coordinator there is nothing to do.
|
||||
if let Some(coordinator) = coordinator.as_mut() {
|
||||
coordinator.wait_for_startup_gate(engine_count, ready_timeout).await?;
|
||||
}
|
||||
|
||||
// 5. After the optional gate has opened, every engine may now send READY.
|
||||
while engines.values().any(|state| !state.is_ready_received()) {
|
||||
debug!(
|
||||
handshake_address,
|
||||
connected = engines.len(),
|
||||
ready = engines.values().filter(|state| state.is_ready_received()).count(),
|
||||
waiting_for = engine_count,
|
||||
"waiting for engine READY"
|
||||
);
|
||||
let message = timeout(ready_timeout, handshake_socket.recv()).await.map_err(|_| {
|
||||
Error::HandshakeTimeout {
|
||||
stage: "READY",
|
||||
timeout: ready_timeout,
|
||||
}
|
||||
})??;
|
||||
let (engine_id, handshake_message) = decode_handshake_message(message, None)?;
|
||||
match handshake_message.status.as_deref() {
|
||||
Some("READY") => {
|
||||
let state = match engines.get_mut(&engine_id) {
|
||||
Some(state) if !state.is_ready_received() => state,
|
||||
_ => {
|
||||
bail_unexpected_handshake_message!(
|
||||
"received READY for unexpected or duplicate engine id {engine_id:?}"
|
||||
);
|
||||
}
|
||||
};
|
||||
debug!(
|
||||
handshake_address,
|
||||
?engine_id,
|
||||
?handshake_message,
|
||||
"received READY from engine"
|
||||
);
|
||||
*state = EngineStartupState::ReadyReceived;
|
||||
}
|
||||
Some("HELLO") => {
|
||||
bail_unexpected_handshake_message!(
|
||||
"received duplicate HELLO for engine id {engine_id:?} after INIT phase completed"
|
||||
);
|
||||
}
|
||||
other => {
|
||||
bail_unexpected_handshake_message!("unexpected handshake status {other:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Wait for every engine to connect to the shared input socket and register itself. The
|
||||
// `ready_response` is a placeholder; it is populated for each engine by
|
||||
// `wait_for_input_registrations` below.
|
||||
let mut engines: Vec<_> = engines
|
||||
.into_keys()
|
||||
.map(|engine_id| ConnectedEngine {
|
||||
engine_id,
|
||||
ready_response: None,
|
||||
})
|
||||
.collect();
|
||||
|
||||
wait_for_input_registrations(&mut input_socket, &mut engines, ready_timeout).await?;
|
||||
debug!(
|
||||
engine_count = engines.len(),
|
||||
"all engines registered on shared input socket"
|
||||
);
|
||||
|
||||
info!(engine_count = engines.len(), "engines connected");
|
||||
|
||||
let (input_send, _) = input_socket.split();
|
||||
|
||||
Ok(ConnectedTransport {
|
||||
input_address,
|
||||
output_address,
|
||||
input_send,
|
||||
output_socket,
|
||||
engines,
|
||||
coordinator,
|
||||
})
|
||||
}
|
||||
|
||||
/// Bind to Python-supplied frontend transport addresses and wait for
|
||||
/// already-initialized engines to register themselves on the input socket.
|
||||
///
|
||||
/// This path mirrors Python's externally managed `AsyncMPClient` bootstrap
|
||||
/// model: the addresses are already fixed by the supervisor, and engine
|
||||
/// identities are synthesized from contiguous rank order instead of being
|
||||
/// discovered through a Rust-owned handshake.
|
||||
pub async fn connect_bootstrapped(
|
||||
input_address: &str,
|
||||
output_address: &str,
|
||||
engine_count: usize,
|
||||
ready_timeout: Duration,
|
||||
) -> Result<ConnectedTransport> {
|
||||
if engine_count == 0 {
|
||||
bail_unexpected_handshake_message!("expected engine_count >= 1");
|
||||
}
|
||||
|
||||
let mut input_socket = RouterSocket::new();
|
||||
let input_address = input_socket.bind(input_address).await?.to_string();
|
||||
|
||||
let mut output_socket = PullSocket::new();
|
||||
let output_address = output_socket.bind(output_address).await?.to_string();
|
||||
|
||||
// TODO: follow start rank
|
||||
let mut engines = (0..engine_count)
|
||||
.map(|index| ConnectedEngine {
|
||||
engine_id: EngineId::from((index as u16).to_le_bytes().to_vec()),
|
||||
ready_response: None,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
wait_for_input_registrations(&mut input_socket, &mut engines, ready_timeout).await?;
|
||||
info!(
|
||||
engine_count = engines.len(),
|
||||
"bootstrapped engines connected"
|
||||
);
|
||||
|
||||
let (input_send, _) = input_socket.split();
|
||||
|
||||
Ok(ConnectedTransport {
|
||||
input_address,
|
||||
output_address,
|
||||
engines,
|
||||
coordinator: None,
|
||||
input_send,
|
||||
output_socket,
|
||||
})
|
||||
}
|
||||
|
||||
/// Bind new input and output sockets.
|
||||
async fn bind_local_sockets(
|
||||
local_host: &str,
|
||||
local_input_address: Option<&str>,
|
||||
local_output_address: Option<&str>,
|
||||
) -> Result<(String, RouterSocket, String, PullSocket)> {
|
||||
let mut input_socket = RouterSocket::new();
|
||||
let input_bind_address = local_input_address
|
||||
.map(str::to_owned)
|
||||
.unwrap_or_else(|| format!("tcp://{local_host}:0"));
|
||||
let input_address = input_socket.bind(&input_bind_address).await?.to_string();
|
||||
|
||||
let mut output_socket = PullSocket::new();
|
||||
let output_bind_address = local_output_address
|
||||
.map(str::to_owned)
|
||||
.unwrap_or_else(|| format!("tcp://{local_host}:0"));
|
||||
let output_address = output_socket.bind(&output_bind_address).await?.to_string();
|
||||
|
||||
Ok((input_address, input_socket, output_address, output_socket))
|
||||
}
|
||||
|
||||
/// Decode a handshake message and validate its structure and identity.
|
||||
fn decode_handshake_message(
|
||||
message: ZmqMessage,
|
||||
expected_id: Option<&EngineId>,
|
||||
) -> Result<(EngineId, ReadyMessage)> {
|
||||
if message.len() != 2 {
|
||||
bail_unexpected_handshake_message!("expected 2 frames, got {}", message.len());
|
||||
}
|
||||
|
||||
let frames = message.into_vec();
|
||||
let actual_id = EngineId(frames[0].clone());
|
||||
if let Some(expected_id) = expected_id
|
||||
&& actual_id != *expected_id
|
||||
{
|
||||
return Err(Error::UnexpectedHandshakeIdentity {
|
||||
expected: expected_id.to_vec(),
|
||||
actual: actual_id.to_vec(),
|
||||
});
|
||||
}
|
||||
|
||||
let handshake_message: ReadyMessage = decode_msgpack(&frames[1])?;
|
||||
Ok((actual_id, handshake_message))
|
||||
}
|
||||
|
||||
/// Send an INIT message to the engine with the local socket addresses for the
|
||||
/// engine to connect to, using the handshake socket.
|
||||
async fn send_init_message(
|
||||
handshake_socket: &mut RouterSocket,
|
||||
engine_id: &EngineId,
|
||||
input_address: &str,
|
||||
output_address: &str,
|
||||
coordinator: Option<&CoordinatorBootstrap>,
|
||||
) -> Result<()> {
|
||||
let init_message = HandshakeInitMessage {
|
||||
addresses: HandshakeAddresses {
|
||||
inputs: vec![input_address.to_string()],
|
||||
outputs: vec![output_address.to_string()],
|
||||
coordinator_input: coordinator.map(|c| c.input_address.clone()),
|
||||
coordinator_output: coordinator.map(|c| c.output_address.clone()),
|
||||
frontend_stats_publish_address: None,
|
||||
},
|
||||
parallel_config: Default::default(),
|
||||
};
|
||||
let payload = encode_msgpack(&init_message)?;
|
||||
let message = ZmqMessage::try_from(vec![engine_id.to_frame(), Bytes::from(payload)])
|
||||
.expect("handshake router messages must contain identity and payload");
|
||||
handshake_socket.send(message).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Receive the input registration message from each engine and validate its
|
||||
/// identity.
|
||||
///
|
||||
/// Each registration contains 2 frames: `[identity, ready-payload]`.
|
||||
///
|
||||
/// Since vLLM commit `c8d98f81f676552c263f35bbde55e6edbe81b4e8` ("[Core]
|
||||
/// Simplify API server handshake"), the payload is a msgpack-encoded
|
||||
/// [`EngineCoreReadyResponse`] carrying post-initialization values such as
|
||||
/// `max_model_len`.
|
||||
///
|
||||
/// Older engines sent an empty second frame here just to establish the
|
||||
/// ROUTER/DEALER backchannel, with no structured payload on the input socket.
|
||||
/// We continue to tolerate that legacy shape so the frontend can still connect
|
||||
/// to slightly older local engine checkouts.
|
||||
async fn wait_for_input_registrations(
|
||||
input_socket: &mut RouterSocket,
|
||||
engines: &mut [ConnectedEngine],
|
||||
ready_timeout: Duration,
|
||||
) -> Result<()> {
|
||||
let mut pending = engines.iter().map(|e| e.engine_id.clone()).collect::<BTreeSet<_>>();
|
||||
|
||||
while !pending.is_empty() {
|
||||
let registration = timeout(ready_timeout, input_socket.recv()).await.map_err(|_| {
|
||||
Error::InputRegistrationTimeout {
|
||||
timeout: ready_timeout,
|
||||
}
|
||||
})??;
|
||||
|
||||
if registration.len() != 2 {
|
||||
bail_unexpected_handshake_message!(
|
||||
"expected 2 frames for engine input registration, got {}",
|
||||
registration.len()
|
||||
);
|
||||
}
|
||||
|
||||
let frames = registration.into_vec();
|
||||
let actual_id = EngineId(frames[0].clone());
|
||||
if !pending.remove(&actual_id) {
|
||||
bail_unexpected_handshake_message!(
|
||||
"received input registration for unexpected engine id {actual_id:?}"
|
||||
);
|
||||
}
|
||||
|
||||
let ready_response = if frames[1].is_empty() {
|
||||
debug!(
|
||||
?actual_id,
|
||||
"received legacy empty input registration from engine"
|
||||
);
|
||||
None
|
||||
} else {
|
||||
let ready_response: EngineCoreReadyResponse = decode_msgpack(&frames[1])?;
|
||||
debug!(
|
||||
?actual_id,
|
||||
?ready_response,
|
||||
"received input registration from engine"
|
||||
);
|
||||
Some(ready_response)
|
||||
};
|
||||
|
||||
// Store the ready response in the corresponding engine entry.
|
||||
if let Some(engine) = engines.iter_mut().find(|e| e.engine_id == actual_id) {
|
||||
engine.ready_response = ready_response;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Send an encoded message to the engine through the input socket.
|
||||
pub async fn send_message(
|
||||
input_send: &mut RouterSendHalf,
|
||||
engine_id: &EngineId,
|
||||
request_type: Bytes,
|
||||
payload: Vec<u8>,
|
||||
) -> Result<()> {
|
||||
let message = ZmqMessage::try_from(vec![
|
||||
engine_id.to_frame(),
|
||||
request_type,
|
||||
Bytes::from(payload),
|
||||
])
|
||||
.expect("router messages must contain identity and payload");
|
||||
|
||||
trace!(
|
||||
?engine_id,
|
||||
frame_count = message.len(),
|
||||
"sending ZMQ message"
|
||||
);
|
||||
input_send.send(message).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Run the output loop to receive messages from the engine and send them to the
|
||||
/// provided channel.
|
||||
pub async fn run_output_loop(
|
||||
mut output_socket: PullSocket,
|
||||
tx: mpsc::Sender<Result<EngineCoreOutputs>>,
|
||||
) {
|
||||
loop {
|
||||
let message = match output_socket.recv().await {
|
||||
Ok(message) => message,
|
||||
Err(error) => {
|
||||
// If we fail to receive a message from the engine, it's likely that the engine
|
||||
// has crashed or become unreachable, so we should notify the
|
||||
// client and shut down the output loop.
|
||||
error!(error = %error.as_report(), "failed to receive output message");
|
||||
let _ = tx.send(Err(Error::Transport(error))).await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let frame_count = message.len();
|
||||
trace!(frame_count, "received output message");
|
||||
let frames = message.into_vec();
|
||||
let frame = frames.first().expect("output message must have at least one frame");
|
||||
let frame_len = frame.len();
|
||||
if frame.as_ref() == ENGINE_CORE_DEAD_SENTINEL {
|
||||
warn!("received ENGINE_CORE_DEAD sentinel from engine");
|
||||
let _ = tx.send(Err(Error::EngineCoreDead)).await;
|
||||
return;
|
||||
}
|
||||
let decoded = match decode_engine_core_outputs(&frames) {
|
||||
Ok(decoded) => {
|
||||
trace!(frame_len, outputs = ?decoded, "decoded output message");
|
||||
Ok(decoded)
|
||||
}
|
||||
Err(error) => {
|
||||
// If we fail to decode the message from the engine, notify the client but keep
|
||||
// the output loop running to continue processing future
|
||||
// messages from the engine.
|
||||
warn!(frame_len, error = %error.as_report(), "failed to decode output message");
|
||||
Err(error)
|
||||
}
|
||||
};
|
||||
|
||||
if tx.send(decoded).await.is_err() {
|
||||
// If we fail to send the decoded message to the client, it's likely that the
|
||||
// client has shut down, so we should shut down the output loop as
|
||||
// well.
|
||||
warn!("output loop rx dropped, shutting down output loop");
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::bind_local_sockets;
|
||||
|
||||
#[tokio::test]
|
||||
async fn bind_local_sockets_resolves_zero_port_bindings() {
|
||||
let (input_address, _input_socket, output_address, _output_socket) =
|
||||
bind_local_sockets("127.0.0.1", None, None).await.expect("bind local sockets");
|
||||
|
||||
assert!(input_address.starts_with("tcp://127.0.0.1:"));
|
||||
assert!(output_address.starts_with("tcp://127.0.0.1:"));
|
||||
assert_ne!(input_address, output_address);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user