[Rust Frontend] Move code from vllm-frontend-rs (#43283)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Signed-off-by: Eric Curtin <eric.curtin@docker.com>
Signed-off-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com>
Signed-off-by: Will.hou <1205157517@qq.com>
Signed-off-by: Will.hou <willamhou@ceresman.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Eric Curtin <eric.curtin@docker.com>
Co-authored-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com>
Co-authored-by: Will.hou <1205157517@qq.com>
Co-authored-by: Will.hou <willamhou@ceresman.com>

Please see https://github.com/Inferact/vllm-frontend-rs for full original commit history.
This commit is contained in:
Bugen Zhao
2026-05-21 17:21:48 -07:00
committed by GitHub
co-authored by Nick Hill Eric Curtin Dev-X25874 Will.hou Will.hou
parent 39d5fa96a7
commit 39910f2b25
284 changed files with 72569 additions and 90 deletions
+38
View File
@@ -0,0 +1,38 @@
[package]
name = "vllm-llm"
version.workspace = true
edition.workspace = true
license.workspace = true
[features]
test-util = []
[dependencies]
easy-ext.workspace = true
enum-as-inner.workspace = true
futures.workspace = true
serde.workspace = true
serde_json.workspace = true
thiserror.workspace = true
tokio.workspace = true
tokio-util.workspace = true
tracing.workspace = true
uuid.workspace = true
vllm-engine-core-client.workspace = true
vllm-metrics.workspace = true
[dev-dependencies]
anyhow.workspace = true
bytes.workspace = true
clap.workspace = true
expect-test.workspace = true
rmp-serde.workspace = true
tokio.workspace = true
tracing-subscriber.workspace = true
uuid.workspace = true
vllm-engine-core-client = { workspace = true, features = ["test-util"] }
vllm-metrics.workspace = true
zeromq.workspace = true
[lints]
workspace = true
+29
View File
@@ -0,0 +1,29 @@
# LLM Smoke Test
Start headless `vllm`:
```bash
source ../vllm/.venv/bin/activate
HF_HUB_OFFLINE=1 \
VLLM_LOGGING_LEVEL=DEBUG \
VLLM_CPU_KVCACHE_SPACE=2 \
VLLM_HOST_IP=127.0.0.1 \
VLLM_LOOPBACK_IP=127.0.0.1 \
python3 -m vllm.entrypoints.cli.main serve Qwen/Qwen3-0.6B \
--headless \
--data-parallel-address 127.0.0.1 \
--data-parallel-rpc-port 62100 \
--data-parallel-size-local 1 \
--max-model-len 512 \
--dtype float16
```
Run the Rust smoke test through the `vllm-llm` generate interface:
```bash
cargo run -p vllm-llm --example external_engine_smoke -- \
--handshake-address tcp://127.0.0.1:62100 \
--host 127.0.0.1
```
IMPORTANT: You must restart `vllm` each time you run the smoke test, as the vLLM engine cannot manage frontend closures and subsequent reconnects. In other words, do not reuse existing `vllm` instances, if any.
@@ -0,0 +1,144 @@
use std::time::Duration;
use anyhow::{Context, Result, bail};
use clap::Parser;
use futures::StreamExt as _;
use tokio::time::timeout;
use tracing_subscriber::EnvFilter;
use vllm_engine_core_client::protocol::EngineCoreSamplingParams;
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, TransportMode};
use vllm_llm::{FinishReason, GenerateOutputStream, GenerateRequest, Llm};
const PROMPT_TOKEN_IDS: &[u32] = &[20841, 448, 6896, 25, 23811];
#[derive(Debug, Parser)]
#[command(about = "Smoke-test the Rust LLM facade against an external vLLM engine.")]
struct Args {
#[arg(long)]
handshake_address: String,
#[arg(long, default_value_t = 1)]
engine_count: usize,
#[arg(long, default_value = "Qwen/Qwen3-0.6B")]
model: String,
#[arg(long, default_value = "127.0.0.1")]
host: String,
#[arg(long, default_value_t = 0)]
client_index: u32,
#[arg(long, default_value_t = 30)]
ready_timeout_secs: u64,
#[arg(long, default_value_t = 120)]
output_timeout_secs: u64,
#[arg(long, default_value_t = 5)]
max_tokens: u32,
}
fn unique_request_id() -> String {
format!("rust-llm-smoke-{}", uuid::Uuid::new_v4())
}
fn init_tracing() {
let filter = EnvFilter::try_from_default_env()
.unwrap_or_else(|_| EnvFilter::new("vllm_engine_core_client=debug"));
let _ = tracing_subscriber::fmt().with_env_filter(filter).try_init();
}
fn build_request(request_id: String, max_tokens: u32) -> GenerateRequest {
GenerateRequest {
request_id,
prompt_token_ids: PROMPT_TOKEN_IDS.to_vec(),
sampling_params: EngineCoreSamplingParams {
max_tokens,
..EngineCoreSamplingParams::for_test()
},
mm_features: None,
arrival_time: None,
cache_salt: None,
trace_headers: None,
priority: 0,
data_parallel_rank: None,
reasoning_ended: None,
lora_request: None,
}
}
#[derive(Debug)]
struct CompletedRequest {
token_ids: Vec<u32>,
finish_reason: FinishReason,
}
async fn wait_for_request_completion(mut stream: GenerateOutputStream) -> Result<CompletedRequest> {
let output = match stream.next().await {
Some(output) => output.context("failed to receive request output")?,
None => bail!("request stream ended without a final output"),
};
let none = stream.next().await;
assert!(
none.is_none(),
"expected final-only stream to end after the final output"
);
let finish_reason = output.finish_reason.expect("final-only output must have a finish reason");
let token_ids = output.token_ids;
Ok(CompletedRequest {
token_ids,
finish_reason,
})
}
async fn wait_for_timeout(
stream: GenerateOutputStream,
output_timeout: Duration,
) -> Result<CompletedRequest> {
timeout(output_timeout, wait_for_request_completion(stream))
.await
.context("timed out waiting for request output")?
}
#[tokio::main(flavor = "multi_thread")]
async fn main() -> Result<()> {
init_tracing();
let args = Args::parse();
let ready_timeout = Duration::from_secs(args.ready_timeout_secs);
let output_timeout = Duration::from_secs(args.output_timeout_secs);
let request_id = unique_request_id();
let client = EngineCoreClient::connect(EngineCoreClientConfig {
transport_mode: TransportMode::HandshakeOwner {
handshake_address: args.handshake_address.clone(),
advertised_host: args.host.clone(),
engine_count: args.engine_count,
ready_timeout,
local_input_address: None,
local_output_address: None,
},
coordinator_mode: None,
model_name: args.model.clone(),
client_index: args.client_index,
})
.await
.context("failed to connect to external vLLM engine")?;
println!("model={}", args.model);
println!("handshake_address={}", args.handshake_address);
println!("engine_count={}", args.engine_count);
println!("input_address={}", client.input_address());
println!("output_address={}", client.output_address());
println!("engine_identities={:x?}", client.engine_identities());
let llm = Llm::new(client);
let request = build_request(request_id.clone(), args.max_tokens);
println!("request_id={request_id}");
println!("prompt_token_ids={PROMPT_TOKEN_IDS:?}");
let stream = llm.generate(request).await.context("failed to submit generate request")?;
let output = wait_for_timeout(stream, output_timeout).await?;
llm.shutdown().await.context("failed to shut down llm client")?;
println!("token_ids={:?}", output.token_ids);
println!("finish_reason={:?}", output.finish_reason);
Ok(())
}
+12
View File
@@ -0,0 +1,12 @@
use thiserror::Error;
pub type Result<T> = std::result::Result<T, Error>;
/// Public error type for the Rust `llm` facade.
#[derive(Debug, Error)]
pub enum Error {
#[error("generate request `{request_id}` has an empty prompt_token_ids")]
EmptyPromptTokenIds { request_id: String },
#[error("engine-core error")]
EngineCoreClient(#[from] vllm_engine_core_client::Error),
}
+100
View File
@@ -0,0 +1,100 @@
use tracing::Span;
use vllm_engine_core_client::EngineCoreClient;
mod error;
mod log_stats;
mod output;
mod request;
mod request_metrics;
pub use error::{Error, Result};
pub use output::{
CollectedGenerateOutput, FinishReason, GenerateOutput, GenerateOutputStream,
GenerateOutputStreamExt, GeneratePromptInfo,
};
pub use request::GenerateRequest;
pub use vllm_engine_core_client::protocol::logprobs::{Logprobs, PositionLogprobs, TokenLogprob};
use crate::log_stats::StatsLogger;
use crate::request_metrics::RequestMetricsTracker;
/// Thin generate-only facade over [`EngineCoreClient`].
///
/// This mirrors the narrow public shape of Python `AsyncLLM.generate()` and
/// `abort()`, but keeps the boundary close to raw engine-core requests and
/// outputs.
pub struct Llm {
client: EngineCoreClient,
randomize_request_id: bool,
stats_logger: Option<StatsLogger>,
}
impl Llm {
/// Create a new minimal LLM facade from an already connected engine-core
/// client.
pub fn new(client: EngineCoreClient) -> Self {
Self {
client,
randomize_request_id: true,
stats_logger: None,
}
}
/// Enable or disable periodic stats logging.
pub fn with_log_stats(mut self, enabled: bool) -> Self {
if enabled {
let stats_logger = StatsLogger::start(
self.client.model_name().to_string(),
self.client.engine_count(),
);
self.stats_logger = Some(stats_logger);
} else {
self.stats_logger = None;
}
self
}
/// Control whether external request ids are randomized before reaching
/// engine-core.
pub fn with_request_id_randomization(mut self, enabled: bool) -> Self {
self.randomize_request_id = enabled;
self
}
/// Expose the underlying engine-core client for low-level utility/admin
/// calls.
pub fn engine_core_client(&self) -> &EngineCoreClient {
&self.client
}
/// Submit one tokenized generate request and return a per-request output
/// stream.
pub async fn generate(&self, req: GenerateRequest) -> Result<GenerateOutputStream> {
let prepared = req.prepare(self.randomize_request_id)?;
let prompt_token_ids = prepared.prompt_token_ids().into();
// Record internal engine-core request ID in the current tracing span.
Span::current().record("engine_request_id", &prepared.engine_request.request_id);
let request_metrics = RequestMetricsTracker::new(
self.client.model_name().to_string(),
prepared.engine_request.arrival_time,
prepared.prompt_token_ids().len() as u32,
(prepared.engine_request.sampling_params.as_ref()).map(|p| p.max_tokens),
1,
);
let stream = self.client.call(prepared.engine_request).await?;
Ok(GenerateOutputStream::new(
prompt_token_ids,
stream,
request_metrics,
))
}
/// Shut down the underlying engine-core client and its background tasks.
pub async fn shutdown(self) -> Result<()> {
self.client.shutdown().await?;
Ok(())
}
}
+199
View File
@@ -0,0 +1,199 @@
use std::fmt::Write;
use std::time::{Duration, Instant};
use tokio_util::task::AbortOnDropHandle;
use tracing::{debug, info};
use vllm_metrics::{
EngineLabels, F64Gauge, METRICS, PromptTokenSourceLabels, U64Counter, U64Gauge,
};
const LOG_STATS_INTERVAL: Duration = Duration::from_secs(10);
/// Cached, cloned metric handles for one engine. Each clone shares the same
/// underlying `Arc<Atomic*>` as the prometheus `Family` entry, so reads go
/// straight to the atomic with no lock.
struct EngineMetrics {
// Counters for throughput deltas.
prompt_tokens_computed: U64Counter,
generation_tokens: U64Counter,
prefix_cache_queries: U64Counter,
prefix_cache_hits: U64Counter,
// Gauges for instantaneous scheduler state.
scheduler_running: U64Gauge,
scheduler_waiting: U64Gauge,
kv_cache_usage: F64Gauge,
}
/// Accumulated snapshot values from the last logging interval, used to compute
/// deltas.
struct CounterSnapshot {
prompt_tokens: u64,
generation_tokens: u64,
prefix_cache_queries: u64,
prefix_cache_hits: u64,
}
/// Periodic stats logger that mirrors Python vLLM's `LoggingStatLogger`.
///
/// Spawns a background task that logs throughput and scheduler state at a fixed
/// interval. When idle (both current and previous throughputs are zero), logs
/// at DEBUG level. When load drops to zero, emits one final INFO-level line
/// before going quiet.
pub(crate) struct StatsLogger {
_task: AbortOnDropHandle<()>,
}
impl StatsLogger {
/// Start the background stats logging task.
pub(crate) fn start(model_name: String, engine_count: usize) -> Self {
let task = AbortOnDropHandle::new(tokio::spawn(async move {
run_stats_logger(model_name, engine_count).await;
}));
Self { _task: task }
}
}
/// Resolve and clone all metric handles once so the hot path is lock-free.
fn resolve_engine_metrics(model_name: &str, engine_count: usize) -> Vec<EngineMetrics> {
let m = &METRICS;
(0..engine_count as u32)
.map(|engine| {
let el = EngineLabels {
model_name: model_name.to_string(),
engine,
};
let pt = PromptTokenSourceLabels {
model_name: model_name.to_string(),
engine,
source: "local_compute",
};
EngineMetrics {
// Use "local_compute" source for prompt throughput (excludes
// cached/transferred tokens), matching Python's
// `iteration_stats.prompt_token_stats.computed`.
prompt_tokens_computed: m.request.prompt_tokens_by_source.get_or_create_owned(&pt),
generation_tokens: m.request.generation_tokens.get_or_create_owned(&el),
prefix_cache_queries: m.scheduler.prefix_cache_queries.get_or_create_owned(&el),
prefix_cache_hits: m.scheduler.prefix_cache_hits.get_or_create_owned(&el),
scheduler_running: m.scheduler.scheduler_running.get_or_create_owned(&el),
scheduler_waiting: m.scheduler.scheduler_waiting.get_or_create_owned(&el),
kv_cache_usage: m.scheduler.kv_cache_usage.get_or_create_owned(&el),
}
})
.collect()
}
async fn run_stats_logger(model_name: String, engine_count: usize) {
let engines = resolve_engine_metrics(&model_name, engine_count);
let mut interval = tokio::time::interval(LOG_STATS_INTERVAL);
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
// The first tick fires immediately; skip it so the first log is after one full
// interval.
interval.tick().await;
let mut prev = read_counters(&engines);
let mut last_log_time = Instant::now();
let mut last_prompt_throughput: f64 = 0.0;
let mut last_generation_throughput: f64 = 0.0;
let mut msg = String::new();
loop {
interval.tick().await;
let now = Instant::now();
let elapsed = now.duration_since(last_log_time).as_secs_f64();
if elapsed <= 0.0 {
continue;
}
let curr = read_counters(&engines);
let prompt_throughput =
curr.prompt_tokens.wrapping_sub(prev.prompt_tokens) as f64 / elapsed;
let generation_throughput =
curr.generation_tokens.wrapping_sub(prev.generation_tokens) as f64 / elapsed;
// Idle = both current and previous throughputs are zero.
let is_idle = prompt_throughput == 0.0
&& generation_throughput == 0.0
&& last_prompt_throughput == 0.0
&& last_generation_throughput == 0.0;
// Read scheduler gauges (aggregate across engines).
let (num_running, num_waiting, kv_cache_usage) = read_scheduler_gauges(&engines);
// Compute prefix cache hit rate over this interval.
let delta_queries = curr.prefix_cache_queries.wrapping_sub(prev.prefix_cache_queries);
let prefix_cache_hit_rate = if delta_queries > 0 {
let delta_hits = curr.prefix_cache_hits.wrapping_sub(prev.prefix_cache_hits);
delta_hits as f64 / delta_queries as f64 * 100.0
} else {
0.0
};
// Build the log line.
msg.clear();
write!(
msg,
"Avg prompt tput: {prompt_throughput:.1} toks/s, \
Avg generation tput: {generation_throughput:.1} toks/s, \
Reqs Running: {num_running}, \
Waiting: {num_waiting}, \
GPU KV cache used: {:.1}%, \
Prefix cache hit rate: {prefix_cache_hit_rate:.1}%",
kv_cache_usage * 100.0,
)
.unwrap();
if is_idle {
debug!("{msg}");
} else {
info!("{msg}");
}
last_prompt_throughput = prompt_throughput;
last_generation_throughput = generation_throughput;
last_log_time = now;
prev = curr;
}
}
/// Read the current cumulative counter values for throughput computation.
fn read_counters(engines: &[EngineMetrics]) -> CounterSnapshot {
let mut snap = CounterSnapshot {
prompt_tokens: 0,
generation_tokens: 0,
prefix_cache_queries: 0,
prefix_cache_hits: 0,
};
for e in engines {
snap.prompt_tokens += e.prompt_tokens_computed.get();
snap.generation_tokens += e.generation_tokens.get();
snap.prefix_cache_queries += e.prefix_cache_queries.get();
snap.prefix_cache_hits += e.prefix_cache_hits.get();
}
snap
}
/// Read the current scheduler gauge values, aggregated across engines.
fn read_scheduler_gauges(engines: &[EngineMetrics]) -> (u64, u64, f64) {
let mut num_running = 0u64;
let mut num_waiting = 0u64;
let mut kv_cache_usage_sum = 0.0f64;
for e in engines {
num_running += e.scheduler_running.get();
num_waiting += e.scheduler_waiting.get();
kv_cache_usage_sum += e.kv_cache_usage.get();
}
let kv_cache_usage = if !engines.is_empty() {
kv_cache_usage_sum / engines.len() as f64
} else {
0.0
};
(num_running, num_waiting, kv_cache_usage)
}
+346
View File
@@ -0,0 +1,346 @@
use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll, ready};
use enum_as_inner::EnumAsInner;
use futures::stream::FusedStream;
use futures::{Stream, StreamExt as _, pin_mut};
use serde::{Deserialize, Serialize};
use vllm_engine_core_client::protocol::logprobs::Logprobs;
use vllm_engine_core_client::protocol::{EngineCoreFinishReason, StopReason};
use vllm_engine_core_client::{AbortCause, EngineCoreOutputStream};
use crate::error::Result;
use crate::request_metrics::{RequestMetricsTracker, current_unix_timestamp_secs};
/// Final raw token output plus terminal stream metadata.
#[derive(Debug, Clone, PartialEq)]
pub struct CollectedGenerateOutput {
pub request_id: String,
pub prompt_token_ids: Vec<u32>,
pub prompt_logprobs: Option<Logprobs>,
pub token_ids: Vec<u32>,
pub logprobs: Option<Logprobs>,
pub finish_reason: FinishReason,
/// Connector-specific KV transfer parameters for disaggregated serving.
pub kv_transfer_params: Option<serde_json::Value>,
}
/// Prompt-scoped metadata emitted only once on the first [`GenerateOutput`] for
/// one request.
#[derive(Debug, Clone, PartialEq)]
pub struct GeneratePromptInfo {
/// Original prompt token IDs for this request.
pub prompt_token_ids: Arc<[u32]>,
/// Prompt logprobs returned by engine-core for scored prompt positions,
/// when requested.
pub prompt_logprobs: Option<Logprobs>,
}
/// The reason a request finished.
///
/// This is a higher-level abstraction over engine-core's finish and stop
/// reasons.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, EnumAsInner)]
pub enum FinishReason {
/// Generation stopped for a stop string, stop token, or EOS.
///
/// The inner stop reason is present for explicit stop strings or stop
/// tokens, and absent for EOS-driven stops.
Stop(Option<StopReason>),
/// `max_tokens` or `max_model_len` was reached.
Length,
/// The request was aborted by the client.
Abort,
/// A retryable request-level internal error occurred.
Error,
/// A repetitive token pattern was detected.
Repetition,
}
impl FinishReason {
/// Construct a stop finish reason caused by EOS rather than an explicit
/// stop string/token.
pub fn stop_eos() -> Self {
Self::Stop(None)
}
/// Returns a human-readable string for this finish reason, used for metrics
/// and reporting.
pub fn as_str(&self) -> &'static str {
match self {
Self::Stop(_) => "stop",
Self::Length => "length",
Self::Abort => "abort",
Self::Error => "error",
Self::Repetition => "repetition",
}
}
/// If this is a stop finish reason, returns the inner stop reason if it
/// exists.
pub fn as_stop_reason(&self) -> Option<&StopReason> {
match self {
Self::Stop(stop_reason) => stop_reason.as_ref(),
_ => None,
}
}
/// If this is a stop finish reason, returns the inner stop reason if it
/// exists.
pub fn into_stop_reason(self) -> Option<StopReason> {
match self {
Self::Stop(stop_reason) => stop_reason,
_ => None,
}
}
}
fn finish_reason_from_engine(
finish_reason: Option<EngineCoreFinishReason>,
stop_reason: Option<StopReason>,
) -> Option<FinishReason> {
finish_reason.map(|reason| match reason {
EngineCoreFinishReason::Stop => FinishReason::Stop(stop_reason),
EngineCoreFinishReason::Length => FinishReason::Length,
EngineCoreFinishReason::Abort => FinishReason::Abort,
EngineCoreFinishReason::Error => FinishReason::Error,
EngineCoreFinishReason::Repetition => FinishReason::Repetition,
})
}
/// Token and logprob output item returned by [`GenerateOutputStream`].
///
/// Original Python output reference:
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/outputs.py#L85-L143>
#[derive(Debug, Clone, PartialEq)]
pub struct GenerateOutput {
/// Unique ID of the request that produced this output.
pub request_id: String,
/// One-time prompt metadata emitted only on the first output for this
/// request.
pub prompt_info: Option<GeneratePromptInfo>,
/// Newly produced token IDs for this step.
pub token_ids: Vec<u32>,
/// Sample logprobs for the generated positions in this step.
pub logprobs: Option<Logprobs>,
/// Terminal finish reason, when this is the final output for the request.
pub finish_reason: Option<FinishReason>,
/// Connector-specific KV transfer parameters for disaggregated serving.
pub kv_transfer_params: Option<serde_json::Value>,
}
impl GenerateOutput {
/// Returns the prompt token IDs when this output carries
/// [`GeneratePromptInfo`].
///
/// Only the first output for a request can return `Some`; all later outputs
/// return `None`.
pub fn prompt_token_ids(&self) -> Option<&Arc<[u32]>> {
self.prompt_info.as_ref().map(|info| &info.prompt_token_ids)
}
/// Returns the prompt logprobs when this output carries
/// [`GeneratePromptInfo`].
///
/// Only the first output for a request can return `Some`; all later outputs
/// return `None`.
pub fn prompt_logprobs(&self) -> Option<&Logprobs> {
self.prompt_info.as_ref().and_then(|info| info.prompt_logprobs.as_ref())
}
/// Returns whether this output is terminal for the request.
pub fn finished(&self) -> bool {
self.finish_reason.is_some()
}
}
#[cfg(any(test, feature = "test-util"))]
impl GenerateOutput {
/// Build a [`GenerateOutput`] for tests.
pub fn for_test(
prompt_token_ids: Option<Arc<[u32]>>,
token_ids: Vec<u32>,
finish_reason: Option<FinishReason>,
) -> Self {
Self {
request_id: String::new(),
prompt_info: prompt_token_ids.map(|ids| GeneratePromptInfo {
prompt_token_ids: ids,
prompt_logprobs: None,
}),
token_ids,
logprobs: None,
finish_reason,
kv_transfer_params: None,
}
}
}
/// Stream of per-request generate outputs for one request.
///
/// - A normal termination of the stream represents a clean completion of the request.
/// - For errors, unexpected closes, or explicit aborts, the stream terminates with an error.
pub struct GenerateOutputStream {
pending_prompt_info: Option<GeneratePromptInfo>,
raw_stream: EngineCoreOutputStream,
request_metrics: RequestMetricsTracker,
}
impl GenerateOutputStream {
/// Create a new generate output stream by adapting one raw engine-core
/// output stream.
pub(crate) fn new(
prompt_token_ids: Arc<[u32]>,
raw_stream: EngineCoreOutputStream,
request_metrics: RequestMetricsTracker,
) -> Self {
Self {
pending_prompt_info: Some(GeneratePromptInfo {
prompt_token_ids,
prompt_logprobs: None,
}),
raw_stream,
request_metrics,
}
}
/// Return the internal engine request ID bound to this stream.
pub fn request_id(&self) -> &str {
self.raw_stream.request_id()
}
}
impl Stream for GenerateOutputStream {
type Item = Result<GenerateOutput>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
let raw = match ready!(Pin::new(&mut self.raw_stream).poll_next(cx)) {
Some(Ok(raw)) => raw,
Some(Err(error)) => return Poll::Ready(Some(Err(error.into()))),
None => return Poll::Ready(None),
};
let received_at = current_unix_timestamp_secs();
self.request_metrics.observe_output(
raw.engine_index,
raw.timestamp,
received_at,
&raw.output,
);
let raw = raw.output;
// Populate the one-time prompt info on the first output.
if let Some(info) = &mut self.pending_prompt_info
&& info.prompt_logprobs.is_none()
{
info.prompt_logprobs =
raw.new_prompt_logprobs_tensors.map(|value| value.into_direct().unwrap());
}
let logprobs = raw.new_logprobs.map(|value| value.into_direct().unwrap());
let finish_reason = finish_reason_from_engine(raw.finish_reason, raw.stop_reason);
if let Some(finish_reason) = finish_reason.as_ref() {
self.request_metrics.record_finished(received_at, finish_reason.clone());
}
let output = GenerateOutput {
request_id: raw.request_id,
prompt_info: self.pending_prompt_info.take(),
token_ids: raw.new_token_ids,
logprobs,
finish_reason,
kv_transfer_params: raw.kv_transfer_params,
};
Poll::Ready(Some(Ok(output)))
}
}
impl FusedStream for GenerateOutputStream {
fn is_terminated(&self) -> bool {
self.raw_stream.is_terminated()
}
}
impl Drop for GenerateOutputStream {
fn drop(&mut self) {
if self.raw_stream.is_terminated() {
// Already terminated cleanly, no need to record abort metrics.
return;
}
// If the user or the upper layer drops a live generate stream,
// `EngineCoreOutputStream::Drop` will trigger an engine-side abort. Record the
// matching terminal request metrics here so frontend-driven aborts are still
// visible as `finished_reason=...` instead of disappearing from observability
// entirely.
let finish_reason = match AbortCause::current() {
AbortCause::DroppedStream => FinishReason::Abort,
AbortCause::StopStringMatched => FinishReason::Stop(None),
};
self.request_metrics
.record_finished(current_unix_timestamp_secs(), finish_reason);
}
}
#[allow(clippy::manual_async_fn, reason = "specify `Send` bound")]
#[easy_ext::ext(GenerateOutputStreamExt)]
impl<T: Stream<Item = Result<GenerateOutput>> + Send> T {
/// Collect the raw generate stream to completion and return the final token
/// output.
pub fn collect_output(self) -> impl Future<Output = Result<CollectedGenerateOutput>> + Send {
async move {
let stream = self;
pin_mut!(stream);
let mut prompt_token_ids = None;
let mut prompt_logprobs = None;
let mut collected: Option<CollectedGenerateOutput> = None;
while let Some(output) = stream.next().await.transpose()? {
if let Some(info) = output.prompt_info {
if prompt_token_ids.is_none() {
prompt_token_ids = Some(info.prompt_token_ids.to_vec());
}
if prompt_logprobs.is_none() {
prompt_logprobs = info.prompt_logprobs;
}
}
if let Some(existing) = collected.as_mut() {
existing.token_ids.extend(output.token_ids);
if let Some(step_logprobs) = output.logprobs {
if let Some(collected_logprobs) = existing.logprobs.as_mut() {
collected_logprobs.positions.extend(step_logprobs.positions);
} else {
existing.logprobs = Some(step_logprobs);
}
}
} else {
collected = Some(CollectedGenerateOutput {
request_id: output.request_id,
prompt_token_ids: prompt_token_ids.take().unwrap_or_default(),
prompt_logprobs: prompt_logprobs.take(),
token_ids: output.token_ids,
logprobs: output.logprobs,
finish_reason: FinishReason::Error,
kv_transfer_params: None,
});
}
if let Some(finish_reason) = output.finish_reason {
let mut collected = collected.expect("terminal output must exist");
collected.finish_reason = finish_reason;
collected.kv_transfer_params = output.kv_transfer_params;
return Ok(collected);
}
}
unreachable!("generate stream should yield an error instead of closing early")
}
}
}
+201
View File
@@ -0,0 +1,201 @@
use std::collections::BTreeMap;
use std::time::{SystemTime, UNIX_EPOCH};
use uuid::Uuid;
use vllm_engine_core_client::protocol::multimodal::MmFeatures;
use vllm_engine_core_client::protocol::{EngineCoreRequest, EngineCoreSamplingParams, OpaqueValue};
use crate::error::{Error, Result};
/// Tokenized decoder-only generate request accepted by [`crate::Llm`].
///
/// This is the first-stage Rust subset of the inputs that eventually flow into
/// Python `AsyncLLM.generate()`. The boundary is intentionally above
/// [`EngineCoreRequest`], but below higher-level text and multimodal
/// preprocessing.
///
/// Original Python API reference:
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/engine/protocol.py#L67-L84>
#[derive(Debug, Clone, PartialEq)]
pub struct GenerateRequest {
/// Unique ID of the request.
pub request_id: String,
/// Token IDs of the prompt.
pub prompt_token_ids: Vec<u32>,
/// Sampling parameters forwarded to engine-core.
pub sampling_params: EngineCoreSamplingParams,
/// Optional multimodal features already prepared by `vllm-chat`.
pub mm_features: Option<MmFeatures>,
// Fields below are currently likely unused by callers.
pub arrival_time: Option<f64>,
pub cache_salt: Option<String>,
pub trace_headers: Option<BTreeMap<String, String>>,
pub priority: i32,
pub data_parallel_rank: Option<u32>,
pub reasoning_ended: Option<bool>,
pub lora_request: Option<OpaqueValue>,
}
#[derive(Debug)]
pub(crate) struct PreparedGenerateRequest {
pub engine_request: EngineCoreRequest,
}
impl GenerateRequest {
/// Validate and lower this request into the raw engine-core request format.
pub(crate) fn prepare(self, randomize_request_id: bool) -> Result<PreparedGenerateRequest> {
if self.prompt_token_ids.is_empty() {
return Err(Error::EmptyPromptTokenIds {
request_id: self.request_id,
});
}
let GenerateRequest {
request_id,
prompt_token_ids,
sampling_params,
mm_features,
arrival_time,
cache_salt,
trace_headers,
priority,
data_parallel_rank,
reasoning_ended,
lora_request,
} = self;
let external_request_id = request_id;
let engine_request_id = if randomize_request_id {
let random_suffix = Uuid::new_v4().simple().to_string();
format!("{external_request_id}-{}", &random_suffix[..8])
} else {
external_request_id.clone()
};
Ok(PreparedGenerateRequest {
engine_request: EngineCoreRequest {
request_id: engine_request_id,
prompt_token_ids: Some(prompt_token_ids),
mm_features,
sampling_params: Some(sampling_params),
pooling_params: None,
arrival_time: arrival_time.unwrap_or_else(current_unix_timestamp_secs),
lora_request,
cache_salt,
data_parallel_rank,
prompt_embeds: None,
prompt_is_token_ids: None,
client_index: 0,
current_wave: 0,
priority,
trace_headers,
resumable: false,
external_req_id: Some(external_request_id),
reasoning_ended,
reasoning_parser_kwargs: None,
abort_immediately: false,
},
})
}
}
impl PreparedGenerateRequest {
/// Return the original prompt token IDs copied into the raw engine request.
pub fn prompt_token_ids(&self) -> &[u32] {
self.engine_request
.prompt_token_ids
.as_ref()
.expect("prepared request must have prompt token ids")
}
}
fn current_unix_timestamp_secs() -> f64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("system clock is before unix epoch")
.as_secs_f64()
}
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
use vllm_engine_core_client::protocol::EngineCoreSamplingParams;
use super::GenerateRequest;
use crate::error::Error;
fn sample_request() -> GenerateRequest {
GenerateRequest {
request_id: "req-1".to_string(),
prompt_token_ids: vec![11, 22, 33],
sampling_params: EngineCoreSamplingParams::for_test(),
mm_features: None,
arrival_time: Some(42.5),
cache_salt: Some("salt".to_string()),
trace_headers: Some(BTreeMap::from([(
"x-trace-id".to_string(),
"abc".to_string(),
)])),
priority: 3,
data_parallel_rank: Some(2),
reasoning_ended: Some(true),
lora_request: None,
}
}
#[test]
fn prepare_builds_engine_core_request() {
let prepared = sample_request().prepare(true).unwrap();
assert_eq!(prepared.prompt_token_ids(), &[11, 22, 33]);
let request = prepared.engine_request;
assert_eq!(request.external_req_id.as_deref(), Some("req-1"));
assert!(request.request_id.starts_with("req-1-"));
assert_ne!(request.request_id, "req-1");
assert_eq!(request.prompt_token_ids.as_deref(), Some(&[11, 22, 33][..]));
assert_eq!(request.arrival_time, 42.5);
assert_eq!(request.cache_salt.as_deref(), Some("salt"));
assert_eq!(request.data_parallel_rank, Some(2));
assert_eq!(
request.trace_headers,
Some(BTreeMap::from([(
"x-trace-id".to_string(),
"abc".to_string(),
)]))
);
assert_eq!(request.reasoning_ended, Some(true));
}
#[test]
fn prepare_rejects_empty_prompt_tokens() {
let mut request = sample_request();
request.prompt_token_ids.clear();
let error = request.prepare(true).unwrap_err();
assert!(matches!(
error,
Error::EmptyPromptTokenIds { request_id } if request_id == "req-1"
));
}
#[test]
fn prepare_can_preserve_external_request_id() {
let prepared = sample_request().prepare(false).unwrap();
let request = prepared.engine_request;
assert_eq!(request.external_req_id.as_deref(), Some("req-1"));
assert_eq!(request.request_id, "req-1");
}
#[test]
fn prepare_forwards_multimodal_features() {
let mut request = sample_request();
request.mm_features = Some(Vec::new());
let prepared = request.prepare(false).unwrap();
assert_eq!(prepared.engine_request.mm_features, Some(Vec::new()));
}
}
+391
View File
@@ -0,0 +1,391 @@
use std::time::{SystemTime, UNIX_EPOCH};
use vllm_engine_core_client::protocol::stats::PrefillStats;
use vllm_engine_core_client::protocol::{EngineCoreEvent, EngineCoreEventType, EngineCoreOutput};
use vllm_metrics::{
EngineLabels, FinishedReasonLabels, METRICS, PromptTokenSourceLabels, RequestMetrics,
};
use crate::FinishReason;
fn metrics() -> &'static RequestMetrics {
&METRICS.request
}
const PROMPT_TOKEN_SOURCE_LOCAL_COMPUTE: &str = "local_compute";
const PROMPT_TOKEN_SOURCE_LOCAL_CACHE_HIT: &str = "local_cache_hit";
const PROMPT_TOKEN_SOURCE_EXTERNAL_KV_TRANSFER: &str = "external_kv_transfer";
/// Request-scoped metrics state tracked across streamed engine-core updates.
///
/// This is the Rust-side counterpart of the Python frontend's request-lifecycle
/// bookkeeping, centered on `RequestStateStats` and the per-output/per-finished
/// update flow.
///
/// Original Python definitions:
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/metrics/stats.py#L200-L237>
///
/// Original Python update flow:
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/engine/output_processor.py#L600-L677>
#[derive(Debug, Clone)]
pub(crate) struct RequestMetricsTracker {
model_name: String,
arrival_time: f64,
prompt_len: u32,
max_tokens_param: Option<u32>,
n_param: u32,
is_prefilling: bool,
queued_ts: f64,
scheduled_ts: f64,
first_token_ts: f64,
last_token_ts: f64,
first_token_latency: f64,
num_generation_tokens: u32,
latest_num_cached_tokens: u32,
last_seen_engine_index: u32,
}
impl RequestMetricsTracker {
/// Create the per-request tracker from the normalized `llm`-layer request
/// context.
pub(crate) fn new(
model_name: String,
arrival_time: f64,
prompt_len: u32,
max_tokens_param: Option<u32>,
n_param: u32,
) -> Self {
Self {
model_name,
arrival_time,
prompt_len,
max_tokens_param,
n_param,
is_prefilling: true,
queued_ts: 0.0,
scheduled_ts: 0.0,
first_token_ts: 0.0,
last_token_ts: 0.0,
first_token_latency: 0.0,
num_generation_tokens: 0,
latest_num_cached_tokens: 0,
last_seen_engine_index: 0,
}
}
/// Update request-lifecycle state from one engine-core output item.
///
/// Original Python stats logic:
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/metrics/stats.py#L331-L384>
pub(crate) fn observe_output(
&mut self,
engine_index: u32,
batch_timestamp: f64,
received_at: f64,
output: &EngineCoreOutput,
) {
self.last_seen_engine_index = engine_index;
if let Some(prefill_stats) = &output.prefill_stats {
self.latest_num_cached_tokens = prefill_stats.num_cached_tokens;
}
self.num_generation_tokens += output.new_token_ids.len() as u32;
metrics()
.generation_tokens
.get_or_create(&engine_labels(&self.model_name, engine_index))
.inc_by(output.new_token_ids.len() as u64);
if let Some(events) = &output.events {
self.observe_events(engine_index, events);
}
if self.is_prefilling {
if let Some(prefill_stats) = &output.prefill_stats {
record_prompt_tokens(&self.model_name, engine_index, prefill_stats);
}
self.first_token_latency = received_at - self.arrival_time;
observe_time_to_first_token_seconds(
&self.model_name,
engine_index,
self.first_token_latency,
);
self.first_token_ts = batch_timestamp;
self.is_prefilling = false;
} else if self.last_token_ts > 0.0 {
observe_inter_token_latency_seconds(
&self.model_name,
engine_index,
batch_timestamp - self.last_token_ts,
);
}
self.last_token_ts = batch_timestamp;
}
/// Emit the terminal request metrics once a finished output has been
/// observed.
///
/// Original Python finished-request stats:
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/metrics/stats.py#L222-L237>
pub(crate) fn record_finished(&self, received_at: f64, finish_reason: FinishReason) {
let labels = engine_labels(&self.model_name, self.last_seen_engine_index);
let prefill_kv_computed_tokens =
self.prompt_len.saturating_sub(self.latest_num_cached_tokens);
let e2e_latency_seconds = received_at - self.arrival_time;
let queue_time_seconds = diff_or_zero(self.scheduled_ts, self.queued_ts);
let prefill_time_seconds = diff_or_zero(self.first_token_ts, self.scheduled_ts);
let decode_time_seconds = diff_or_zero(self.last_token_ts, self.first_token_ts);
let inference_time_seconds = diff_or_zero(self.last_token_ts, self.scheduled_ts);
let time_per_output_token_seconds = if self.num_generation_tokens > 1 {
diff_or_zero(self.last_token_ts, self.first_token_ts)
/ (self.num_generation_tokens - 1) as f64
} else {
0.0
};
record_request_success(&self.model_name, self.last_seen_engine_index, finish_reason);
metrics()
.request_prompt_tokens
.get_or_create(&labels)
.observe(self.prompt_len as f64);
metrics()
.request_generation_tokens
.get_or_create(&labels)
.observe(self.num_generation_tokens as f64);
metrics()
.request_max_num_generation_tokens
.get_or_create(&labels)
.observe(self.num_generation_tokens as f64);
if let Some(max_tokens_param) = self.max_tokens_param {
metrics()
.request_params_max_tokens
.get_or_create(&labels)
.observe(max_tokens_param as f64);
}
metrics().request_params_n.get_or_create(&labels).observe(self.n_param as f64);
metrics()
.request_prefill_kv_computed_tokens
.get_or_create(&labels)
.observe(prefill_kv_computed_tokens as f64);
metrics()
.e2e_request_latency_seconds
.get_or_create(&labels)
.observe(e2e_latency_seconds);
metrics()
.request_queue_time_seconds
.get_or_create(&labels)
.observe(queue_time_seconds);
metrics()
.request_prefill_time_seconds
.get_or_create(&labels)
.observe(prefill_time_seconds);
metrics()
.request_decode_time_seconds
.get_or_create(&labels)
.observe(decode_time_seconds);
metrics()
.request_inference_time_seconds
.get_or_create(&labels)
.observe(inference_time_seconds);
metrics()
.request_time_per_output_token_seconds
.get_or_create(&labels)
.observe(time_per_output_token_seconds);
}
fn observe_events(&mut self, engine_index: u32, events: &[EngineCoreEvent]) {
for event in events {
match event.r#type {
EngineCoreEventType::Queued => {
self.queued_ts = event.timestamp;
}
EngineCoreEventType::Scheduled => {
if self.scheduled_ts == 0.0 {
self.scheduled_ts = event.timestamp;
}
}
EngineCoreEventType::Preempted => {
metrics()
.num_preemptions
.get_or_create(&engine_labels(&self.model_name, engine_index))
.inc();
}
}
}
}
}
fn engine_labels(model_name: &str, engine: u32) -> EngineLabels {
EngineLabels {
model_name: model_name.to_string(),
engine,
}
}
fn observe_time_to_first_token_seconds(model_name: &str, engine: u32, seconds: f64) {
metrics()
.time_to_first_token_seconds
.get_or_create(&engine_labels(model_name, engine))
.observe(seconds);
}
fn observe_inter_token_latency_seconds(model_name: &str, engine: u32, seconds: f64) {
metrics()
.inter_token_latency_seconds
.get_or_create(&engine_labels(model_name, engine))
.observe(seconds);
}
fn record_request_success(model_name: &str, engine: u32, finish_reason: FinishReason) {
metrics()
.request_success
.get_or_create(&FinishedReasonLabels {
model_name: model_name.to_string(),
engine,
finished_reason: finish_reason.as_str(),
})
.inc();
}
fn prompt_token_source_labels(
model_name: &str,
engine: u32,
source: &'static str,
) -> PromptTokenSourceLabels {
PromptTokenSourceLabels {
model_name: model_name.to_string(),
engine,
source,
}
}
fn record_prompt_tokens(model_name: &str, engine: u32, prefill_stats: &PrefillStats) {
let computed = prefill_stats.num_computed_tokens as u64;
let local_cache_hit = prefill_stats.num_local_cached_tokens as u64;
let external_kv_transfer = prefill_stats.num_external_cached_tokens as u64;
metrics()
.prompt_tokens
.get_or_create(&engine_labels(model_name, engine))
.inc_by(prefill_stats.num_prompt_tokens as u64);
metrics()
.prompt_tokens_by_source
.get_or_create(&prompt_token_source_labels(
model_name,
engine,
PROMPT_TOKEN_SOURCE_LOCAL_COMPUTE,
))
.inc_by(computed);
metrics()
.prompt_tokens_by_source
.get_or_create(&prompt_token_source_labels(
model_name,
engine,
PROMPT_TOKEN_SOURCE_LOCAL_CACHE_HIT,
))
.inc_by(local_cache_hit);
metrics()
.prompt_tokens_by_source
.get_or_create(&prompt_token_source_labels(
model_name,
engine,
PROMPT_TOKEN_SOURCE_EXTERNAL_KV_TRANSFER,
))
.inc_by(external_kv_transfer);
metrics()
.prompt_tokens_cached
.get_or_create(&engine_labels(model_name, engine))
.inc_by(prefill_stats.num_cached_tokens as u64);
}
fn diff_or_zero(end: f64, start: f64) -> f64 {
if end > 0.0 && start > 0.0 && end >= start {
end - start
} else {
0.0
}
}
/// Return the current wall-clock time in seconds since the Unix epoch.
///
/// This is used for frontend-side latency measurements such as TTFT and E2E,
/// matching the Python frontend's use of wall-clock request arrival/iteration
/// timestamps rather than engine-core's monotonic scheduler timestamps.
///
/// Original Python request timestamp source:
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/metrics/stats.py#L206-L216>
pub(crate) fn current_unix_timestamp_secs() -> f64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("system clock is before unix epoch")
.as_secs_f64()
}
#[cfg(test)]
mod tests {
use vllm_engine_core_client::protocol::stats::PrefillStats;
use vllm_engine_core_client::protocol::{EngineCoreEvent, EngineCoreEventType};
use super::{RequestMetricsTracker, diff_or_zero};
#[test]
fn tracker_updates_timing_state_across_prefill_decode_and_finish() {
let mut tracker = RequestMetricsTracker::new("model".to_string(), 100.0, 64, Some(128), 1);
tracker.observe_output(
2,
10.0,
100.2,
&vllm_engine_core_client::protocol::EngineCoreOutput {
request_id: "req-1".to_string(),
new_token_ids: vec![1],
finish_reason: None,
events: Some(vec![
EngineCoreEvent {
r#type: EngineCoreEventType::Queued,
timestamp: 8.0,
},
EngineCoreEvent {
r#type: EngineCoreEventType::Scheduled,
timestamp: 9.0,
},
]),
prefill_stats: Some(PrefillStats {
num_prompt_tokens: 64,
num_computed_tokens: 60,
num_cached_tokens: 4,
num_local_cached_tokens: 4,
num_external_cached_tokens: 0,
}),
..Default::default()
},
);
tracker.observe_output(
2,
11.5,
100.4,
&vllm_engine_core_client::protocol::EngineCoreOutput {
request_id: "req-1".to_string(),
new_token_ids: vec![2, 3],
finish_reason: None,
events: Some(vec![EngineCoreEvent {
r#type: EngineCoreEventType::Preempted,
timestamp: 10.5,
}]),
..Default::default()
},
);
assert!(!tracker.is_prefilling);
assert_eq!(tracker.last_seen_engine_index, 2);
assert_eq!(tracker.num_generation_tokens, 3);
assert_eq!(tracker.queued_ts, 8.0);
assert_eq!(tracker.scheduled_ts, 9.0);
assert_eq!(tracker.first_token_ts, 10.0);
assert_eq!(tracker.last_token_ts, 11.5);
assert!((tracker.first_token_latency - 0.2).abs() < 1e-9);
assert_eq!(
diff_or_zero(tracker.last_token_ts, tracker.first_token_ts),
1.5
);
}
}
+749
View File
@@ -0,0 +1,749 @@
use std::collections::BTreeSet;
use std::sync::Once;
use std::time::Duration;
use futures::StreamExt as _;
use tokio::time::timeout;
use tracing_subscriber::EnvFilter;
use uuid::Uuid;
use vllm_engine_core_client::protocol::logprobs::{
Logprobs, MaybeWireLogprobs, PositionLogprobs, TokenLogprob,
};
use vllm_engine_core_client::protocol::stats::PrefillStats;
use vllm_engine_core_client::protocol::{
EngineCoreEvent, EngineCoreEventType, EngineCoreFinishReason, EngineCoreOutput,
EngineCoreOutputs, EngineCoreRequest, EngineCoreSamplingParams,
};
use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig};
use vllm_llm::{
Error, FinishReason, GenerateOutputStreamExt as _, GeneratePromptInfo, GenerateRequest, Llm,
};
use vllm_metrics::METRICS;
use zeromq::prelude::{SocketRecv, SocketSend};
use zeromq::{DealerSocket, PushSocket, ZmqMessage};
static TRACING: Once = Once::new();
fn request_output(
request_id: &str,
new_token_ids: Vec<u32>,
finish_reason: Option<EngineCoreFinishReason>,
) -> EngineCoreOutput {
request_output_with_events(request_id, new_token_ids, finish_reason, None)
}
fn request_output_with_events(
request_id: &str,
new_token_ids: Vec<u32>,
finish_reason: Option<EngineCoreFinishReason>,
events: Option<Vec<EngineCoreEvent>>,
) -> EngineCoreOutput {
EngineCoreOutput {
request_id: request_id.to_string(),
new_token_ids,
new_logprobs: None,
new_prompt_logprobs_tensors: None,
pooling_output: None,
finish_reason,
stop_reason: None,
events,
kv_transfer_params: None,
trace_headers: None,
prefill_stats: None,
routed_experts: None,
num_nans_in_logits: 0,
}
}
fn request_output_with_logprobs(
request_id: &str,
new_token_ids: Vec<u32>,
finish_reason: Option<EngineCoreFinishReason>,
new_logprobs: Option<Logprobs>,
prompt_logprobs: Option<Logprobs>,
) -> EngineCoreOutput {
EngineCoreOutput {
request_id: request_id.to_string(),
new_token_ids,
new_logprobs: new_logprobs.map(MaybeWireLogprobs::Direct),
new_prompt_logprobs_tensors: prompt_logprobs.map(MaybeWireLogprobs::Direct),
pooling_output: None,
finish_reason,
stop_reason: None,
events: None,
kv_transfer_params: None,
trace_headers: None,
prefill_stats: None,
routed_experts: None,
num_nans_in_logits: 0,
}
}
fn request_output_with_logprobs_and_kv(
request_id: &str,
new_token_ids: Vec<u32>,
finish_reason: Option<EngineCoreFinishReason>,
new_logprobs: Option<Logprobs>,
prompt_logprobs: Option<Logprobs>,
kv_transfer_params: Option<serde_json::Value>,
) -> EngineCoreOutput {
EngineCoreOutput {
request_id: request_id.to_string(),
new_token_ids,
new_logprobs: new_logprobs.map(MaybeWireLogprobs::Direct),
new_prompt_logprobs_tensors: prompt_logprobs.map(MaybeWireLogprobs::Direct),
pooling_output: None,
finish_reason,
stop_reason: None,
events: None,
kv_transfer_params,
trace_headers: None,
prefill_stats: None,
routed_experts: None,
num_nans_in_logits: 0,
}
}
fn logprobs_for_position(
sampled_token_id: u32,
sampled_logprob: f32,
sampled_rank: u32,
top_token_id: u32,
top_logprob: f32,
) -> Logprobs {
Logprobs {
positions: vec![PositionLogprobs {
entries: vec![
TokenLogprob {
token_id: sampled_token_id,
logprob: sampled_logprob,
rank: sampled_rank,
},
TokenLogprob {
token_id: top_token_id,
logprob: top_logprob,
rank: 1,
},
],
}],
}
}
fn prompt_logprobs() -> Logprobs {
Logprobs {
positions: vec![
PositionLogprobs {
entries: vec![
TokenLogprob {
token_id: 11,
logprob: -0.1,
rank: 2,
},
TokenLogprob {
token_id: 7,
logprob: -0.05,
rank: 1,
},
],
},
PositionLogprobs {
entries: vec![
TokenLogprob {
token_id: 22,
logprob: -0.2,
rank: 3,
},
TokenLogprob {
token_id: 8,
logprob: -0.1,
rank: 1,
},
],
},
],
}
}
fn sample_generate_request(request_id: &str, max_tokens: u32) -> GenerateRequest {
GenerateRequest {
request_id: request_id.to_string(),
prompt_token_ids: vec![11, 22],
sampling_params: EngineCoreSamplingParams {
max_tokens,
..EngineCoreSamplingParams::for_test()
},
mm_features: None,
arrival_time: Some(42.5),
cache_salt: None,
trace_headers: None,
priority: 0,
data_parallel_rank: None,
reasoning_ended: None,
lora_request: None,
}
}
async fn send_outputs(push: &mut PushSocket, outputs: EngineCoreOutputs) {
push.send(ZmqMessage::from(rmp_serde::to_vec_named(&outputs).unwrap()))
.await
.unwrap();
}
async fn recv_engine_message(dealer: &mut DealerSocket) -> Vec<bytes::Bytes> {
dealer.recv().await.unwrap().into_vec()
}
async fn connect_async_llm_with_ipc(
handshake_address: String,
client_index: u32,
model_name: &str,
ipc: &IpcNamespace,
) -> Llm {
let client = EngineCoreClient::connect(
EngineCoreClientConfig::new_single(handshake_address)
.with_model_name(model_name)
.with_client_index(client_index)
.with_local_input_output_addresses(
Some(ipc.input_endpoint()),
Some(ipc.output_endpoint()),
),
)
.await
.unwrap();
Llm::new(client)
}
fn request_metrics_model_name(prefix: &str) -> String {
format!("{prefix}-{}", Uuid::new_v4().simple())
}
fn init_tracing() {
TRACING.call_once(|| {
let filter = EnvFilter::try_from_default_env()
.unwrap_or_else(|_| EnvFilter::new("vllm_engine_core_client=debug"));
let _ = tracing_subscriber::fmt().with_test_writer().with_env_filter(filter).try_init();
});
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn generate_streams_outputs() {
init_tracing();
let ipc = IpcNamespace::new().unwrap();
let handshake_address = ipc.handshake_endpoint();
let engine_id = b"engine-delta".to_vec();
let (shutdown_tx, engine_task) = spawn_mock_engine_task(
handshake_address.clone(),
engine_id.clone(),
|dealer, push| {
Box::pin(async move {
let add = recv_engine_message(dealer).await;
assert_eq!(add[0].as_ref(), &[0x00]);
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
assert_eq!(request.external_req_id.as_deref(), Some("req-delta"));
assert!(request.request_id.starts_with("req-delta-"));
assert_ne!(request.request_id, "req-delta");
assert_eq!(request.client_index, 7);
assert_eq!(request.prompt_token_ids, Some(vec![11, 22]));
send_outputs(
push,
EngineCoreOutputs {
outputs: vec![
request_output_with_logprobs(
&request.request_id,
vec![1, 2],
None,
Some(logprobs_for_position(1, -0.3, 4, 9, -0.1)),
Some(prompt_logprobs()),
),
request_output_with_logprobs(
&request.request_id,
vec![3],
Some(EngineCoreFinishReason::Length),
Some(logprobs_for_position(3, -0.4, 5, 10, -0.2)),
None,
),
],
finished_requests: Some(BTreeSet::from([request.request_id.clone()])),
..Default::default()
},
)
.await;
})
},
);
let llm = connect_async_llm_with_ipc(handshake_address, 7, "test-model", &ipc).await;
let mut stream = llm.generate(sample_generate_request("req-delta", 3)).await.unwrap();
let internal_id = stream.request_id().to_string();
let first = stream.next().await.unwrap().unwrap();
assert_eq!(first.request_id, internal_id);
assert_eq!(
first.prompt_info,
Some(GeneratePromptInfo {
prompt_token_ids: vec![11, 22].into(),
prompt_logprobs: Some(prompt_logprobs()),
})
);
assert_eq!(first.token_ids, vec![1, 2]);
assert_eq!(
first.logprobs,
Some(logprobs_for_position(1, -0.3, 4, 9, -0.1))
);
assert_eq!(first.finish_reason, None);
let second = stream.next().await.unwrap().unwrap();
assert_eq!(second.prompt_info, None);
assert_eq!(second.token_ids, vec![3]);
assert_eq!(
second.logprobs,
Some(logprobs_for_position(3, -0.4, 5, 10, -0.2))
);
assert_eq!(second.finish_reason, Some(FinishReason::Length));
assert!(stream.next().await.is_none());
let _ = shutdown_tx.send(());
engine_task.await.unwrap();
llm.shutdown().await.unwrap();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn collect_output_aggregates_raw_tokens_logprobs_and_terminal_metadata() {
init_tracing();
let ipc = IpcNamespace::new().unwrap();
let handshake_address = ipc.handshake_endpoint();
let engine_id = b"engine-collect-output".to_vec();
let (shutdown_tx, engine_task) = spawn_mock_engine_task(
handshake_address.clone(),
engine_id.clone(),
|dealer, push| {
Box::pin(async move {
let add = recv_engine_message(dealer).await;
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
assert_eq!(request.external_req_id.as_deref(), Some("req-collect"));
assert!(request.request_id.starts_with("req-collect-"));
send_outputs(
push,
EngineCoreOutputs {
engine_index: 0,
outputs: vec![
request_output_with_logprobs(
&request.request_id,
vec![33],
None,
Some(logprobs_for_position(33, -0.1, 1, 99, -0.2)),
Some(prompt_logprobs()),
),
request_output_with_logprobs_and_kv(
&request.request_id,
vec![44],
Some(EngineCoreFinishReason::Stop),
Some(logprobs_for_position(44, -0.3, 1, 88, -0.4)),
None,
Some(serde_json::json!({"connector": "x"})),
),
],
scheduler_stats: None,
timestamp: 0.0,
utility_output: None,
finished_requests: None,
wave_complete: None,
start_wave: None,
},
)
.await;
})
},
);
let llm = connect_async_llm_with_ipc(handshake_address, 7, "test-model", &ipc).await;
let stream = llm.generate(sample_generate_request("req-collect", 4)).await.unwrap();
let internal_id = stream.request_id().to_string();
let collected = stream.collect_output().await.unwrap();
let _ = shutdown_tx.send(());
engine_task.await.unwrap();
assert_eq!(collected.request_id, internal_id);
assert_eq!(collected.prompt_token_ids, vec![11, 22]);
assert_eq!(collected.token_ids, vec![33, 44]);
assert_eq!(collected.finish_reason, FinishReason::stop_eos());
assert_eq!(collected.prompt_logprobs, Some(prompt_logprobs()));
assert_eq!(
collected.logprobs.as_ref().map(|lp| lp.positions.len()),
Some(2)
);
assert_eq!(
collected.kv_transfer_params,
Some(serde_json::json!({"connector": "x"}))
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn generate_propagates_unexpected_close_errors() {
let ipc = IpcNamespace::new().unwrap();
let handshake_address = ipc.handshake_endpoint();
let engine_id = b"engine-close".to_vec();
let (shutdown_tx, engine_task) = spawn_mock_engine_task(
handshake_address.clone(),
engine_id.clone(),
|dealer, push| {
Box::pin(async move {
let add = recv_engine_message(dealer).await;
assert_eq!(add[0].as_ref(), &[0x00]);
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
send_outputs(
push,
EngineCoreOutputs {
finished_requests: Some(BTreeSet::from([request.request_id])),
..Default::default()
},
)
.await;
})
},
);
let llm = connect_async_llm_with_ipc(handshake_address, 0, "test-model", &ipc).await;
let mut stream = llm.generate(sample_generate_request("req-close", 1)).await.unwrap();
let internal_id = stream.request_id().to_string();
let error = stream.next().await.unwrap().unwrap_err();
assert!(matches!(
error,
Error::EngineCoreClient(vllm_engine_core_client::Error::RequestStreamClosed {
request_id
}) if request_id == internal_id
));
assert!(stream.next().await.is_none());
let _ = shutdown_tx.send(());
engine_task.await.unwrap();
llm.shutdown().await.unwrap();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn dropping_a_live_generate_stream_triggers_abort() {
let ipc = IpcNamespace::new().unwrap();
let handshake_address = ipc.handshake_endpoint();
let engine_id = b"engine-drop".to_vec();
let (shutdown_tx, engine_task) = spawn_mock_engine_task(
handshake_address.clone(),
engine_id.clone(),
|dealer, push| {
Box::pin(async move {
let add = recv_engine_message(dealer).await;
assert_eq!(add[0].as_ref(), &[0x00]);
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
assert_eq!(request.external_req_id.as_deref(), Some("req-drop"));
assert!(request.request_id.starts_with("req-drop-"));
send_outputs(
push,
EngineCoreOutputs {
outputs: vec![request_output(&request.request_id, vec![99], None)],
..Default::default()
},
)
.await;
let abort =
timeout(Duration::from_secs(1), recv_engine_message(dealer)).await.unwrap();
assert_eq!(abort[0].as_ref(), &[0x01]);
let aborted_ids: Vec<String> = rmp_serde::from_slice(&abort[1]).unwrap();
assert_eq!(aborted_ids, vec![request.request_id]);
})
},
);
let llm = connect_async_llm_with_ipc(handshake_address, 0, "test-model", &ipc).await;
let mut stream = llm.generate(sample_generate_request("req-drop", 4)).await.unwrap();
let output = stream.next().await.unwrap().unwrap();
assert_eq!(output.token_ids, vec![99]);
drop(stream);
let _ = shutdown_tx.send(());
engine_task.await.unwrap();
llm.shutdown().await.unwrap();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn duplicate_external_request_ids_are_randomized_before_reaching_engine_core_client() {
let ipc = IpcNamespace::new().unwrap();
let handshake_address = ipc.handshake_endpoint();
let engine_id = b"engine-dup".to_vec();
let (shutdown_tx, engine_task) = spawn_mock_engine_task(
handshake_address.clone(),
engine_id.clone(),
|dealer, push| {
Box::pin(async move {
let add_1 = recv_engine_message(dealer).await;
assert_eq!(add_1[0].as_ref(), &[0x00]);
let request_1: EngineCoreRequest = rmp_serde::from_slice(&add_1[1]).unwrap();
assert_eq!(request_1.external_req_id.as_deref(), Some("req-dup"));
assert!(request_1.request_id.starts_with("req-dup-"));
let add_2 = recv_engine_message(dealer).await;
assert_eq!(add_2[0].as_ref(), &[0x00]);
let request_2: EngineCoreRequest = rmp_serde::from_slice(&add_2[1]).unwrap();
assert_eq!(request_2.external_req_id.as_deref(), Some("req-dup"));
assert!(request_2.request_id.starts_with("req-dup-"));
assert_ne!(request_1.request_id, request_2.request_id);
send_outputs(
push,
EngineCoreOutputs {
outputs: vec![request_output(
&request_1.request_id,
vec![],
Some(EngineCoreFinishReason::Length),
)],
finished_requests: Some(BTreeSet::from([request_1.request_id.clone()])),
..Default::default()
},
)
.await;
send_outputs(
push,
EngineCoreOutputs {
outputs: vec![request_output(
&request_2.request_id,
vec![],
Some(EngineCoreFinishReason::Length),
)],
finished_requests: Some(BTreeSet::from([request_2.request_id])),
..Default::default()
},
)
.await;
})
},
);
let llm = connect_async_llm_with_ipc(handshake_address, 0, "test-model", &ipc).await;
let stream_1 = llm.generate(sample_generate_request("req-dup", 1)).await.unwrap();
let stream_2 = llm.generate(sample_generate_request("req-dup", 1)).await.unwrap();
let internal_id_1 = stream_1.request_id().to_string();
let internal_id_2 = stream_2.request_id().to_string();
let collected_1 = stream_1.collect_output().await.unwrap();
let collected_2 = stream_2.collect_output().await.unwrap();
assert_eq!(collected_1.request_id, internal_id_1);
assert_eq!(collected_2.request_id, internal_id_2);
let _ = shutdown_tx.send(());
engine_task.await.unwrap();
llm.shutdown().await.unwrap();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn generate_records_request_metrics_in_prometheus_output() {
let ipc = IpcNamespace::new().unwrap();
let handshake_address = ipc.handshake_endpoint();
let engine_id = b"engine-metrics".to_vec();
let model_name = request_metrics_model_name("metrics-model");
let (shutdown_tx, engine_task) = spawn_mock_engine_task(
handshake_address.clone(),
engine_id.clone(),
|dealer, push| {
Box::pin(async move {
let add = recv_engine_message(dealer).await;
assert_eq!(add[0].as_ref(), &[0x00]);
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
send_outputs(
push,
EngineCoreOutputs {
engine_index: 4,
timestamp: 10.0,
outputs: vec![EngineCoreOutput {
prefill_stats: Some(PrefillStats {
num_prompt_tokens: 2,
num_computed_tokens: 2,
..Default::default()
}),
..request_output_with_events(
&request.request_id,
vec![1],
None,
Some(vec![
EngineCoreEvent {
r#type: EngineCoreEventType::Queued,
timestamp: 8.0,
},
EngineCoreEvent {
r#type: EngineCoreEventType::Scheduled,
timestamp: 9.0,
},
]),
)
}],
..Default::default()
},
)
.await;
send_outputs(
push,
EngineCoreOutputs {
engine_index: 4,
timestamp: 11.5,
outputs: vec![request_output_with_events(
&request.request_id,
vec![2, 3],
Some(EngineCoreFinishReason::Length),
Some(vec![EngineCoreEvent {
r#type: EngineCoreEventType::Preempted,
timestamp: 10.5,
}]),
)],
finished_requests: Some(BTreeSet::from([request.request_id])),
..Default::default()
},
)
.await;
})
},
);
let llm = connect_async_llm_with_ipc(handshake_address, 0, &model_name, &ipc).await;
let mut request = sample_generate_request("req-metrics", 8);
request.arrival_time = None;
let mut stream = llm.generate(request).await.unwrap();
assert_eq!(stream.next().await.unwrap().unwrap().token_ids, vec![1]);
let final_output = stream.next().await.unwrap().unwrap();
assert_eq!(final_output.token_ids, vec![2, 3]);
assert_eq!(final_output.finish_reason, Some(FinishReason::Length));
assert!(stream.next().await.is_none());
let rendered = METRICS.render().unwrap();
assert!(rendered.contains(&format!(
"vllm:request_success_total{{model_name=\"{model_name}\",engine=\"4\",finished_reason=\"length\"}} 1"
)));
assert!(rendered.contains(&format!(
"vllm:prompt_tokens_total{{model_name=\"{model_name}\",engine=\"4\"}} 2"
)));
assert!(rendered.contains(&format!(
"vllm:prompt_tokens_by_source_total{{model_name=\"{model_name}\",engine=\"4\",source=\"local_compute\"}} 2"
)));
assert!(rendered.contains(&format!(
"vllm:prompt_tokens_by_source_total{{model_name=\"{model_name}\",engine=\"4\",source=\"local_cache_hit\"}} 0"
)));
assert!(rendered.contains(&format!(
"vllm:prompt_tokens_by_source_total{{model_name=\"{model_name}\",engine=\"4\",source=\"external_kv_transfer\"}} 0"
)));
assert!(rendered.contains(&format!(
"vllm:prompt_tokens_cached_total{{model_name=\"{model_name}\",engine=\"4\"}} 0"
)));
assert!(rendered.contains(&format!(
"vllm:generation_tokens_total{{model_name=\"{model_name}\",engine=\"4\"}} 3"
)));
assert!(rendered.contains(&format!(
"vllm:num_preemptions_total{{model_name=\"{model_name}\",engine=\"4\"}} 1"
)));
assert!(rendered.contains(&format!(
"vllm:time_to_first_token_seconds_count{{model_name=\"{model_name}\",engine=\"4\"}} 1"
)));
assert!(rendered.contains(&format!(
"vllm:inter_token_latency_seconds_count{{model_name=\"{model_name}\",engine=\"4\"}} 1"
)));
assert!(rendered.contains(&format!(
"vllm:e2e_request_latency_seconds_count{{model_name=\"{model_name}\",engine=\"4\"}} 1"
)));
assert!(rendered.contains(&format!(
"vllm:request_prompt_tokens_count{{model_name=\"{model_name}\",engine=\"4\"}} 1"
)));
assert!(rendered.contains(&format!(
"vllm:request_generation_tokens_count{{model_name=\"{model_name}\",engine=\"4\"}} 1"
)));
assert!(rendered.contains(&format!(
"vllm:request_prefill_kv_computed_tokens_count{{model_name=\"{model_name}\",engine=\"4\"}} 1"
)));
let _ = shutdown_tx.send(());
engine_task.await.unwrap();
llm.shutdown().await.unwrap();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn dropping_stream_records_abort_terminal_request_metrics() {
let ipc = IpcNamespace::new().unwrap();
let handshake_address = ipc.handshake_endpoint();
let engine_id = b"engine-metrics-drop".to_vec();
let model_name = request_metrics_model_name("metrics-drop-model");
let (shutdown_tx, engine_task) = spawn_mock_engine_task(
handshake_address.clone(),
engine_id.clone(),
|dealer, push| {
Box::pin(async move {
let add = recv_engine_message(dealer).await;
assert_eq!(add[0].as_ref(), &[0x00]);
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
assert_eq!(request.external_req_id.as_deref(), Some("req-metrics-drop"));
assert!(request.request_id.starts_with("req-metrics-drop-"));
send_outputs(
push,
EngineCoreOutputs {
engine_index: 5,
timestamp: 10.0,
outputs: vec![request_output_with_events(
&request.request_id,
vec![99],
None,
Some(vec![
EngineCoreEvent {
r#type: EngineCoreEventType::Queued,
timestamp: 8.0,
},
EngineCoreEvent {
r#type: EngineCoreEventType::Scheduled,
timestamp: 9.0,
},
]),
)],
..Default::default()
},
)
.await;
let abort =
timeout(Duration::from_secs(1), recv_engine_message(dealer)).await.unwrap();
assert_eq!(abort[0].as_ref(), &[0x01]);
let aborted_ids: Vec<String> = rmp_serde::from_slice(&abort[1]).unwrap();
assert_eq!(aborted_ids, vec![request.request_id]);
})
},
);
let llm = connect_async_llm_with_ipc(handshake_address, 0, &model_name, &ipc).await;
let mut request = sample_generate_request("req-metrics-drop", 8);
request.arrival_time = None;
let mut stream = llm.generate(request).await.unwrap();
assert_eq!(stream.next().await.unwrap().unwrap().token_ids, vec![99]);
drop(stream);
let _ = shutdown_tx.send(());
engine_task.await.unwrap();
let rendered = METRICS.render().unwrap();
assert!(rendered.contains(&format!(
"vllm:request_success_total{{model_name=\"{model_name}\",engine=\"5\",finished_reason=\"abort\"}} 1"
)));
assert!(rendered.contains(&format!(
"vllm:e2e_request_latency_seconds_count{{model_name=\"{model_name}\",engine=\"5\"}} 1"
)));
llm.shutdown().await.unwrap();
}