[Rust Frontend] Move code from vllm-frontend-rs (#43283)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Signed-off-by: Eric Curtin <eric.curtin@docker.com> Signed-off-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com> Signed-off-by: Will.hou <1205157517@qq.com> Signed-off-by: Will.hou <willamhou@ceresman.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Eric Curtin <eric.curtin@docker.com> Co-authored-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com> Co-authored-by: Will.hou <1205157517@qq.com> Co-authored-by: Will.hou <willamhou@ceresman.com> Please see https://github.com/Inferact/vllm-frontend-rs for full original commit history.
This commit is contained in:
co-authored by
Nick Hill
Eric Curtin
Dev-X25874
Will.hou
Will.hou
parent
39d5fa96a7
commit
39910f2b25
@@ -0,0 +1,38 @@
|
||||
[package]
|
||||
name = "vllm-llm"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[features]
|
||||
test-util = []
|
||||
|
||||
[dependencies]
|
||||
easy-ext.workspace = true
|
||||
enum-as-inner.workspace = true
|
||||
futures.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
thiserror.workspace = true
|
||||
tokio.workspace = true
|
||||
tokio-util.workspace = true
|
||||
tracing.workspace = true
|
||||
uuid.workspace = true
|
||||
vllm-engine-core-client.workspace = true
|
||||
vllm-metrics.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
anyhow.workspace = true
|
||||
bytes.workspace = true
|
||||
clap.workspace = true
|
||||
expect-test.workspace = true
|
||||
rmp-serde.workspace = true
|
||||
tokio.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
uuid.workspace = true
|
||||
vllm-engine-core-client = { workspace = true, features = ["test-util"] }
|
||||
vllm-metrics.workspace = true
|
||||
zeromq.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -0,0 +1,29 @@
|
||||
# LLM Smoke Test
|
||||
|
||||
Start headless `vllm`:
|
||||
|
||||
```bash
|
||||
source ../vllm/.venv/bin/activate
|
||||
HF_HUB_OFFLINE=1 \
|
||||
VLLM_LOGGING_LEVEL=DEBUG \
|
||||
VLLM_CPU_KVCACHE_SPACE=2 \
|
||||
VLLM_HOST_IP=127.0.0.1 \
|
||||
VLLM_LOOPBACK_IP=127.0.0.1 \
|
||||
python3 -m vllm.entrypoints.cli.main serve Qwen/Qwen3-0.6B \
|
||||
--headless \
|
||||
--data-parallel-address 127.0.0.1 \
|
||||
--data-parallel-rpc-port 62100 \
|
||||
--data-parallel-size-local 1 \
|
||||
--max-model-len 512 \
|
||||
--dtype float16
|
||||
```
|
||||
|
||||
Run the Rust smoke test through the `vllm-llm` generate interface:
|
||||
|
||||
```bash
|
||||
cargo run -p vllm-llm --example external_engine_smoke -- \
|
||||
--handshake-address tcp://127.0.0.1:62100 \
|
||||
--host 127.0.0.1
|
||||
```
|
||||
|
||||
IMPORTANT: You must restart `vllm` each time you run the smoke test, as the vLLM engine cannot manage frontend closures and subsequent reconnects. In other words, do not reuse existing `vllm` instances, if any.
|
||||
@@ -0,0 +1,144 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use clap::Parser;
|
||||
use futures::StreamExt as _;
|
||||
use tokio::time::timeout;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use vllm_engine_core_client::protocol::EngineCoreSamplingParams;
|
||||
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, TransportMode};
|
||||
use vllm_llm::{FinishReason, GenerateOutputStream, GenerateRequest, Llm};
|
||||
|
||||
const PROMPT_TOKEN_IDS: &[u32] = &[20841, 448, 6896, 25, 23811];
|
||||
|
||||
#[derive(Debug, Parser)]
|
||||
#[command(about = "Smoke-test the Rust LLM facade against an external vLLM engine.")]
|
||||
struct Args {
|
||||
#[arg(long)]
|
||||
handshake_address: String,
|
||||
#[arg(long, default_value_t = 1)]
|
||||
engine_count: usize,
|
||||
#[arg(long, default_value = "Qwen/Qwen3-0.6B")]
|
||||
model: String,
|
||||
#[arg(long, default_value = "127.0.0.1")]
|
||||
host: String,
|
||||
#[arg(long, default_value_t = 0)]
|
||||
client_index: u32,
|
||||
#[arg(long, default_value_t = 30)]
|
||||
ready_timeout_secs: u64,
|
||||
#[arg(long, default_value_t = 120)]
|
||||
output_timeout_secs: u64,
|
||||
#[arg(long, default_value_t = 5)]
|
||||
max_tokens: u32,
|
||||
}
|
||||
|
||||
fn unique_request_id() -> String {
|
||||
format!("rust-llm-smoke-{}", uuid::Uuid::new_v4())
|
||||
}
|
||||
|
||||
fn init_tracing() {
|
||||
let filter = EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| EnvFilter::new("vllm_engine_core_client=debug"));
|
||||
let _ = tracing_subscriber::fmt().with_env_filter(filter).try_init();
|
||||
}
|
||||
|
||||
fn build_request(request_id: String, max_tokens: u32) -> GenerateRequest {
|
||||
GenerateRequest {
|
||||
request_id,
|
||||
prompt_token_ids: PROMPT_TOKEN_IDS.to_vec(),
|
||||
sampling_params: EngineCoreSamplingParams {
|
||||
max_tokens,
|
||||
..EngineCoreSamplingParams::for_test()
|
||||
},
|
||||
mm_features: None,
|
||||
arrival_time: None,
|
||||
cache_salt: None,
|
||||
trace_headers: None,
|
||||
priority: 0,
|
||||
data_parallel_rank: None,
|
||||
reasoning_ended: None,
|
||||
lora_request: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct CompletedRequest {
|
||||
token_ids: Vec<u32>,
|
||||
finish_reason: FinishReason,
|
||||
}
|
||||
|
||||
async fn wait_for_request_completion(mut stream: GenerateOutputStream) -> Result<CompletedRequest> {
|
||||
let output = match stream.next().await {
|
||||
Some(output) => output.context("failed to receive request output")?,
|
||||
None => bail!("request stream ended without a final output"),
|
||||
};
|
||||
|
||||
let none = stream.next().await;
|
||||
assert!(
|
||||
none.is_none(),
|
||||
"expected final-only stream to end after the final output"
|
||||
);
|
||||
|
||||
let finish_reason = output.finish_reason.expect("final-only output must have a finish reason");
|
||||
let token_ids = output.token_ids;
|
||||
|
||||
Ok(CompletedRequest {
|
||||
token_ids,
|
||||
finish_reason,
|
||||
})
|
||||
}
|
||||
|
||||
async fn wait_for_timeout(
|
||||
stream: GenerateOutputStream,
|
||||
output_timeout: Duration,
|
||||
) -> Result<CompletedRequest> {
|
||||
timeout(output_timeout, wait_for_request_completion(stream))
|
||||
.await
|
||||
.context("timed out waiting for request output")?
|
||||
}
|
||||
|
||||
#[tokio::main(flavor = "multi_thread")]
|
||||
async fn main() -> Result<()> {
|
||||
init_tracing();
|
||||
let args = Args::parse();
|
||||
let ready_timeout = Duration::from_secs(args.ready_timeout_secs);
|
||||
let output_timeout = Duration::from_secs(args.output_timeout_secs);
|
||||
let request_id = unique_request_id();
|
||||
let client = EngineCoreClient::connect(EngineCoreClientConfig {
|
||||
transport_mode: TransportMode::HandshakeOwner {
|
||||
handshake_address: args.handshake_address.clone(),
|
||||
advertised_host: args.host.clone(),
|
||||
engine_count: args.engine_count,
|
||||
ready_timeout,
|
||||
local_input_address: None,
|
||||
local_output_address: None,
|
||||
},
|
||||
coordinator_mode: None,
|
||||
model_name: args.model.clone(),
|
||||
client_index: args.client_index,
|
||||
})
|
||||
.await
|
||||
.context("failed to connect to external vLLM engine")?;
|
||||
|
||||
println!("model={}", args.model);
|
||||
println!("handshake_address={}", args.handshake_address);
|
||||
println!("engine_count={}", args.engine_count);
|
||||
println!("input_address={}", client.input_address());
|
||||
println!("output_address={}", client.output_address());
|
||||
println!("engine_identities={:x?}", client.engine_identities());
|
||||
|
||||
let llm = Llm::new(client);
|
||||
let request = build_request(request_id.clone(), args.max_tokens);
|
||||
println!("request_id={request_id}");
|
||||
println!("prompt_token_ids={PROMPT_TOKEN_IDS:?}");
|
||||
|
||||
let stream = llm.generate(request).await.context("failed to submit generate request")?;
|
||||
let output = wait_for_timeout(stream, output_timeout).await?;
|
||||
|
||||
llm.shutdown().await.context("failed to shut down llm client")?;
|
||||
|
||||
println!("token_ids={:?}", output.token_ids);
|
||||
println!("finish_reason={:?}", output.finish_reason);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
use thiserror::Error;
|
||||
|
||||
pub type Result<T> = std::result::Result<T, Error>;
|
||||
|
||||
/// Public error type for the Rust `llm` facade.
|
||||
#[derive(Debug, Error)]
|
||||
pub enum Error {
|
||||
#[error("generate request `{request_id}` has an empty prompt_token_ids")]
|
||||
EmptyPromptTokenIds { request_id: String },
|
||||
#[error("engine-core error")]
|
||||
EngineCoreClient(#[from] vllm_engine_core_client::Error),
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
use tracing::Span;
|
||||
use vllm_engine_core_client::EngineCoreClient;
|
||||
|
||||
mod error;
|
||||
mod log_stats;
|
||||
mod output;
|
||||
mod request;
|
||||
mod request_metrics;
|
||||
|
||||
pub use error::{Error, Result};
|
||||
pub use output::{
|
||||
CollectedGenerateOutput, FinishReason, GenerateOutput, GenerateOutputStream,
|
||||
GenerateOutputStreamExt, GeneratePromptInfo,
|
||||
};
|
||||
pub use request::GenerateRequest;
|
||||
pub use vllm_engine_core_client::protocol::logprobs::{Logprobs, PositionLogprobs, TokenLogprob};
|
||||
|
||||
use crate::log_stats::StatsLogger;
|
||||
use crate::request_metrics::RequestMetricsTracker;
|
||||
|
||||
/// Thin generate-only facade over [`EngineCoreClient`].
|
||||
///
|
||||
/// This mirrors the narrow public shape of Python `AsyncLLM.generate()` and
|
||||
/// `abort()`, but keeps the boundary close to raw engine-core requests and
|
||||
/// outputs.
|
||||
pub struct Llm {
|
||||
client: EngineCoreClient,
|
||||
randomize_request_id: bool,
|
||||
stats_logger: Option<StatsLogger>,
|
||||
}
|
||||
|
||||
impl Llm {
|
||||
/// Create a new minimal LLM facade from an already connected engine-core
|
||||
/// client.
|
||||
pub fn new(client: EngineCoreClient) -> Self {
|
||||
Self {
|
||||
client,
|
||||
randomize_request_id: true,
|
||||
stats_logger: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Enable or disable periodic stats logging.
|
||||
pub fn with_log_stats(mut self, enabled: bool) -> Self {
|
||||
if enabled {
|
||||
let stats_logger = StatsLogger::start(
|
||||
self.client.model_name().to_string(),
|
||||
self.client.engine_count(),
|
||||
);
|
||||
self.stats_logger = Some(stats_logger);
|
||||
} else {
|
||||
self.stats_logger = None;
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Control whether external request ids are randomized before reaching
|
||||
/// engine-core.
|
||||
pub fn with_request_id_randomization(mut self, enabled: bool) -> Self {
|
||||
self.randomize_request_id = enabled;
|
||||
self
|
||||
}
|
||||
|
||||
/// Expose the underlying engine-core client for low-level utility/admin
|
||||
/// calls.
|
||||
pub fn engine_core_client(&self) -> &EngineCoreClient {
|
||||
&self.client
|
||||
}
|
||||
|
||||
/// Submit one tokenized generate request and return a per-request output
|
||||
/// stream.
|
||||
pub async fn generate(&self, req: GenerateRequest) -> Result<GenerateOutputStream> {
|
||||
let prepared = req.prepare(self.randomize_request_id)?;
|
||||
let prompt_token_ids = prepared.prompt_token_ids().into();
|
||||
|
||||
// Record internal engine-core request ID in the current tracing span.
|
||||
Span::current().record("engine_request_id", &prepared.engine_request.request_id);
|
||||
|
||||
let request_metrics = RequestMetricsTracker::new(
|
||||
self.client.model_name().to_string(),
|
||||
prepared.engine_request.arrival_time,
|
||||
prepared.prompt_token_ids().len() as u32,
|
||||
(prepared.engine_request.sampling_params.as_ref()).map(|p| p.max_tokens),
|
||||
1,
|
||||
);
|
||||
let stream = self.client.call(prepared.engine_request).await?;
|
||||
|
||||
Ok(GenerateOutputStream::new(
|
||||
prompt_token_ids,
|
||||
stream,
|
||||
request_metrics,
|
||||
))
|
||||
}
|
||||
|
||||
/// Shut down the underlying engine-core client and its background tasks.
|
||||
pub async fn shutdown(self) -> Result<()> {
|
||||
self.client.shutdown().await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
use std::fmt::Write;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use tokio_util::task::AbortOnDropHandle;
|
||||
use tracing::{debug, info};
|
||||
use vllm_metrics::{
|
||||
EngineLabels, F64Gauge, METRICS, PromptTokenSourceLabels, U64Counter, U64Gauge,
|
||||
};
|
||||
|
||||
const LOG_STATS_INTERVAL: Duration = Duration::from_secs(10);
|
||||
|
||||
/// Cached, cloned metric handles for one engine. Each clone shares the same
|
||||
/// underlying `Arc<Atomic*>` as the prometheus `Family` entry, so reads go
|
||||
/// straight to the atomic with no lock.
|
||||
struct EngineMetrics {
|
||||
// Counters for throughput deltas.
|
||||
prompt_tokens_computed: U64Counter,
|
||||
generation_tokens: U64Counter,
|
||||
prefix_cache_queries: U64Counter,
|
||||
prefix_cache_hits: U64Counter,
|
||||
|
||||
// Gauges for instantaneous scheduler state.
|
||||
scheduler_running: U64Gauge,
|
||||
scheduler_waiting: U64Gauge,
|
||||
kv_cache_usage: F64Gauge,
|
||||
}
|
||||
|
||||
/// Accumulated snapshot values from the last logging interval, used to compute
|
||||
/// deltas.
|
||||
struct CounterSnapshot {
|
||||
prompt_tokens: u64,
|
||||
generation_tokens: u64,
|
||||
prefix_cache_queries: u64,
|
||||
prefix_cache_hits: u64,
|
||||
}
|
||||
|
||||
/// Periodic stats logger that mirrors Python vLLM's `LoggingStatLogger`.
|
||||
///
|
||||
/// Spawns a background task that logs throughput and scheduler state at a fixed
|
||||
/// interval. When idle (both current and previous throughputs are zero), logs
|
||||
/// at DEBUG level. When load drops to zero, emits one final INFO-level line
|
||||
/// before going quiet.
|
||||
pub(crate) struct StatsLogger {
|
||||
_task: AbortOnDropHandle<()>,
|
||||
}
|
||||
|
||||
impl StatsLogger {
|
||||
/// Start the background stats logging task.
|
||||
pub(crate) fn start(model_name: String, engine_count: usize) -> Self {
|
||||
let task = AbortOnDropHandle::new(tokio::spawn(async move {
|
||||
run_stats_logger(model_name, engine_count).await;
|
||||
}));
|
||||
Self { _task: task }
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve and clone all metric handles once so the hot path is lock-free.
|
||||
fn resolve_engine_metrics(model_name: &str, engine_count: usize) -> Vec<EngineMetrics> {
|
||||
let m = &METRICS;
|
||||
(0..engine_count as u32)
|
||||
.map(|engine| {
|
||||
let el = EngineLabels {
|
||||
model_name: model_name.to_string(),
|
||||
engine,
|
||||
};
|
||||
let pt = PromptTokenSourceLabels {
|
||||
model_name: model_name.to_string(),
|
||||
engine,
|
||||
source: "local_compute",
|
||||
};
|
||||
EngineMetrics {
|
||||
// Use "local_compute" source for prompt throughput (excludes
|
||||
// cached/transferred tokens), matching Python's
|
||||
// `iteration_stats.prompt_token_stats.computed`.
|
||||
prompt_tokens_computed: m.request.prompt_tokens_by_source.get_or_create_owned(&pt),
|
||||
generation_tokens: m.request.generation_tokens.get_or_create_owned(&el),
|
||||
prefix_cache_queries: m.scheduler.prefix_cache_queries.get_or_create_owned(&el),
|
||||
prefix_cache_hits: m.scheduler.prefix_cache_hits.get_or_create_owned(&el),
|
||||
scheduler_running: m.scheduler.scheduler_running.get_or_create_owned(&el),
|
||||
scheduler_waiting: m.scheduler.scheduler_waiting.get_or_create_owned(&el),
|
||||
kv_cache_usage: m.scheduler.kv_cache_usage.get_or_create_owned(&el),
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn run_stats_logger(model_name: String, engine_count: usize) {
|
||||
let engines = resolve_engine_metrics(&model_name, engine_count);
|
||||
|
||||
let mut interval = tokio::time::interval(LOG_STATS_INTERVAL);
|
||||
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
|
||||
// The first tick fires immediately; skip it so the first log is after one full
|
||||
// interval.
|
||||
interval.tick().await;
|
||||
|
||||
let mut prev = read_counters(&engines);
|
||||
let mut last_log_time = Instant::now();
|
||||
let mut last_prompt_throughput: f64 = 0.0;
|
||||
let mut last_generation_throughput: f64 = 0.0;
|
||||
|
||||
let mut msg = String::new();
|
||||
loop {
|
||||
interval.tick().await;
|
||||
|
||||
let now = Instant::now();
|
||||
let elapsed = now.duration_since(last_log_time).as_secs_f64();
|
||||
if elapsed <= 0.0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
let curr = read_counters(&engines);
|
||||
|
||||
let prompt_throughput =
|
||||
curr.prompt_tokens.wrapping_sub(prev.prompt_tokens) as f64 / elapsed;
|
||||
let generation_throughput =
|
||||
curr.generation_tokens.wrapping_sub(prev.generation_tokens) as f64 / elapsed;
|
||||
|
||||
// Idle = both current and previous throughputs are zero.
|
||||
let is_idle = prompt_throughput == 0.0
|
||||
&& generation_throughput == 0.0
|
||||
&& last_prompt_throughput == 0.0
|
||||
&& last_generation_throughput == 0.0;
|
||||
|
||||
// Read scheduler gauges (aggregate across engines).
|
||||
let (num_running, num_waiting, kv_cache_usage) = read_scheduler_gauges(&engines);
|
||||
|
||||
// Compute prefix cache hit rate over this interval.
|
||||
let delta_queries = curr.prefix_cache_queries.wrapping_sub(prev.prefix_cache_queries);
|
||||
let prefix_cache_hit_rate = if delta_queries > 0 {
|
||||
let delta_hits = curr.prefix_cache_hits.wrapping_sub(prev.prefix_cache_hits);
|
||||
delta_hits as f64 / delta_queries as f64 * 100.0
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
// Build the log line.
|
||||
msg.clear();
|
||||
write!(
|
||||
msg,
|
||||
"Avg prompt tput: {prompt_throughput:.1} toks/s, \
|
||||
Avg generation tput: {generation_throughput:.1} toks/s, \
|
||||
Reqs Running: {num_running}, \
|
||||
Waiting: {num_waiting}, \
|
||||
GPU KV cache used: {:.1}%, \
|
||||
Prefix cache hit rate: {prefix_cache_hit_rate:.1}%",
|
||||
kv_cache_usage * 100.0,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
if is_idle {
|
||||
debug!("{msg}");
|
||||
} else {
|
||||
info!("{msg}");
|
||||
}
|
||||
|
||||
last_prompt_throughput = prompt_throughput;
|
||||
last_generation_throughput = generation_throughput;
|
||||
last_log_time = now;
|
||||
prev = curr;
|
||||
}
|
||||
}
|
||||
|
||||
/// Read the current cumulative counter values for throughput computation.
|
||||
fn read_counters(engines: &[EngineMetrics]) -> CounterSnapshot {
|
||||
let mut snap = CounterSnapshot {
|
||||
prompt_tokens: 0,
|
||||
generation_tokens: 0,
|
||||
prefix_cache_queries: 0,
|
||||
prefix_cache_hits: 0,
|
||||
};
|
||||
for e in engines {
|
||||
snap.prompt_tokens += e.prompt_tokens_computed.get();
|
||||
snap.generation_tokens += e.generation_tokens.get();
|
||||
snap.prefix_cache_queries += e.prefix_cache_queries.get();
|
||||
snap.prefix_cache_hits += e.prefix_cache_hits.get();
|
||||
}
|
||||
snap
|
||||
}
|
||||
|
||||
/// Read the current scheduler gauge values, aggregated across engines.
|
||||
fn read_scheduler_gauges(engines: &[EngineMetrics]) -> (u64, u64, f64) {
|
||||
let mut num_running = 0u64;
|
||||
let mut num_waiting = 0u64;
|
||||
let mut kv_cache_usage_sum = 0.0f64;
|
||||
|
||||
for e in engines {
|
||||
num_running += e.scheduler_running.get();
|
||||
num_waiting += e.scheduler_waiting.get();
|
||||
kv_cache_usage_sum += e.kv_cache_usage.get();
|
||||
}
|
||||
|
||||
let kv_cache_usage = if !engines.is_empty() {
|
||||
kv_cache_usage_sum / engines.len() as f64
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
(num_running, num_waiting, kv_cache_usage)
|
||||
}
|
||||
@@ -0,0 +1,346 @@
|
||||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::task::{Context, Poll, ready};
|
||||
|
||||
use enum_as_inner::EnumAsInner;
|
||||
use futures::stream::FusedStream;
|
||||
use futures::{Stream, StreamExt as _, pin_mut};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use vllm_engine_core_client::protocol::logprobs::Logprobs;
|
||||
use vllm_engine_core_client::protocol::{EngineCoreFinishReason, StopReason};
|
||||
use vllm_engine_core_client::{AbortCause, EngineCoreOutputStream};
|
||||
|
||||
use crate::error::Result;
|
||||
use crate::request_metrics::{RequestMetricsTracker, current_unix_timestamp_secs};
|
||||
|
||||
/// Final raw token output plus terminal stream metadata.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct CollectedGenerateOutput {
|
||||
pub request_id: String,
|
||||
pub prompt_token_ids: Vec<u32>,
|
||||
pub prompt_logprobs: Option<Logprobs>,
|
||||
pub token_ids: Vec<u32>,
|
||||
pub logprobs: Option<Logprobs>,
|
||||
pub finish_reason: FinishReason,
|
||||
/// Connector-specific KV transfer parameters for disaggregated serving.
|
||||
pub kv_transfer_params: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
/// Prompt-scoped metadata emitted only once on the first [`GenerateOutput`] for
|
||||
/// one request.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct GeneratePromptInfo {
|
||||
/// Original prompt token IDs for this request.
|
||||
pub prompt_token_ids: Arc<[u32]>,
|
||||
/// Prompt logprobs returned by engine-core for scored prompt positions,
|
||||
/// when requested.
|
||||
pub prompt_logprobs: Option<Logprobs>,
|
||||
}
|
||||
|
||||
/// The reason a request finished.
|
||||
///
|
||||
/// This is a higher-level abstraction over engine-core's finish and stop
|
||||
/// reasons.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, EnumAsInner)]
|
||||
pub enum FinishReason {
|
||||
/// Generation stopped for a stop string, stop token, or EOS.
|
||||
///
|
||||
/// The inner stop reason is present for explicit stop strings or stop
|
||||
/// tokens, and absent for EOS-driven stops.
|
||||
Stop(Option<StopReason>),
|
||||
/// `max_tokens` or `max_model_len` was reached.
|
||||
Length,
|
||||
/// The request was aborted by the client.
|
||||
Abort,
|
||||
/// A retryable request-level internal error occurred.
|
||||
Error,
|
||||
/// A repetitive token pattern was detected.
|
||||
Repetition,
|
||||
}
|
||||
|
||||
impl FinishReason {
|
||||
/// Construct a stop finish reason caused by EOS rather than an explicit
|
||||
/// stop string/token.
|
||||
pub fn stop_eos() -> Self {
|
||||
Self::Stop(None)
|
||||
}
|
||||
|
||||
/// Returns a human-readable string for this finish reason, used for metrics
|
||||
/// and reporting.
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Stop(_) => "stop",
|
||||
Self::Length => "length",
|
||||
Self::Abort => "abort",
|
||||
Self::Error => "error",
|
||||
Self::Repetition => "repetition",
|
||||
}
|
||||
}
|
||||
|
||||
/// If this is a stop finish reason, returns the inner stop reason if it
|
||||
/// exists.
|
||||
pub fn as_stop_reason(&self) -> Option<&StopReason> {
|
||||
match self {
|
||||
Self::Stop(stop_reason) => stop_reason.as_ref(),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// If this is a stop finish reason, returns the inner stop reason if it
|
||||
/// exists.
|
||||
pub fn into_stop_reason(self) -> Option<StopReason> {
|
||||
match self {
|
||||
Self::Stop(stop_reason) => stop_reason,
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn finish_reason_from_engine(
|
||||
finish_reason: Option<EngineCoreFinishReason>,
|
||||
stop_reason: Option<StopReason>,
|
||||
) -> Option<FinishReason> {
|
||||
finish_reason.map(|reason| match reason {
|
||||
EngineCoreFinishReason::Stop => FinishReason::Stop(stop_reason),
|
||||
EngineCoreFinishReason::Length => FinishReason::Length,
|
||||
EngineCoreFinishReason::Abort => FinishReason::Abort,
|
||||
EngineCoreFinishReason::Error => FinishReason::Error,
|
||||
EngineCoreFinishReason::Repetition => FinishReason::Repetition,
|
||||
})
|
||||
}
|
||||
|
||||
/// Token and logprob output item returned by [`GenerateOutputStream`].
|
||||
///
|
||||
/// Original Python output reference:
|
||||
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/outputs.py#L85-L143>
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct GenerateOutput {
|
||||
/// Unique ID of the request that produced this output.
|
||||
pub request_id: String,
|
||||
/// One-time prompt metadata emitted only on the first output for this
|
||||
/// request.
|
||||
pub prompt_info: Option<GeneratePromptInfo>,
|
||||
/// Newly produced token IDs for this step.
|
||||
pub token_ids: Vec<u32>,
|
||||
/// Sample logprobs for the generated positions in this step.
|
||||
pub logprobs: Option<Logprobs>,
|
||||
/// Terminal finish reason, when this is the final output for the request.
|
||||
pub finish_reason: Option<FinishReason>,
|
||||
/// Connector-specific KV transfer parameters for disaggregated serving.
|
||||
pub kv_transfer_params: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
impl GenerateOutput {
|
||||
/// Returns the prompt token IDs when this output carries
|
||||
/// [`GeneratePromptInfo`].
|
||||
///
|
||||
/// Only the first output for a request can return `Some`; all later outputs
|
||||
/// return `None`.
|
||||
pub fn prompt_token_ids(&self) -> Option<&Arc<[u32]>> {
|
||||
self.prompt_info.as_ref().map(|info| &info.prompt_token_ids)
|
||||
}
|
||||
|
||||
/// Returns the prompt logprobs when this output carries
|
||||
/// [`GeneratePromptInfo`].
|
||||
///
|
||||
/// Only the first output for a request can return `Some`; all later outputs
|
||||
/// return `None`.
|
||||
pub fn prompt_logprobs(&self) -> Option<&Logprobs> {
|
||||
self.prompt_info.as_ref().and_then(|info| info.prompt_logprobs.as_ref())
|
||||
}
|
||||
|
||||
/// Returns whether this output is terminal for the request.
|
||||
pub fn finished(&self) -> bool {
|
||||
self.finish_reason.is_some()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
impl GenerateOutput {
|
||||
/// Build a [`GenerateOutput`] for tests.
|
||||
pub fn for_test(
|
||||
prompt_token_ids: Option<Arc<[u32]>>,
|
||||
token_ids: Vec<u32>,
|
||||
finish_reason: Option<FinishReason>,
|
||||
) -> Self {
|
||||
Self {
|
||||
request_id: String::new(),
|
||||
prompt_info: prompt_token_ids.map(|ids| GeneratePromptInfo {
|
||||
prompt_token_ids: ids,
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
token_ids,
|
||||
logprobs: None,
|
||||
finish_reason,
|
||||
kv_transfer_params: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Stream of per-request generate outputs for one request.
|
||||
///
|
||||
/// - A normal termination of the stream represents a clean completion of the request.
|
||||
/// - For errors, unexpected closes, or explicit aborts, the stream terminates with an error.
|
||||
pub struct GenerateOutputStream {
|
||||
pending_prompt_info: Option<GeneratePromptInfo>,
|
||||
raw_stream: EngineCoreOutputStream,
|
||||
request_metrics: RequestMetricsTracker,
|
||||
}
|
||||
|
||||
impl GenerateOutputStream {
|
||||
/// Create a new generate output stream by adapting one raw engine-core
|
||||
/// output stream.
|
||||
pub(crate) fn new(
|
||||
prompt_token_ids: Arc<[u32]>,
|
||||
raw_stream: EngineCoreOutputStream,
|
||||
request_metrics: RequestMetricsTracker,
|
||||
) -> Self {
|
||||
Self {
|
||||
pending_prompt_info: Some(GeneratePromptInfo {
|
||||
prompt_token_ids,
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
raw_stream,
|
||||
request_metrics,
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the internal engine request ID bound to this stream.
|
||||
pub fn request_id(&self) -> &str {
|
||||
self.raw_stream.request_id()
|
||||
}
|
||||
}
|
||||
|
||||
impl Stream for GenerateOutputStream {
|
||||
type Item = Result<GenerateOutput>;
|
||||
|
||||
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
let raw = match ready!(Pin::new(&mut self.raw_stream).poll_next(cx)) {
|
||||
Some(Ok(raw)) => raw,
|
||||
Some(Err(error)) => return Poll::Ready(Some(Err(error.into()))),
|
||||
None => return Poll::Ready(None),
|
||||
};
|
||||
|
||||
let received_at = current_unix_timestamp_secs();
|
||||
self.request_metrics.observe_output(
|
||||
raw.engine_index,
|
||||
raw.timestamp,
|
||||
received_at,
|
||||
&raw.output,
|
||||
);
|
||||
|
||||
let raw = raw.output;
|
||||
|
||||
// Populate the one-time prompt info on the first output.
|
||||
if let Some(info) = &mut self.pending_prompt_info
|
||||
&& info.prompt_logprobs.is_none()
|
||||
{
|
||||
info.prompt_logprobs =
|
||||
raw.new_prompt_logprobs_tensors.map(|value| value.into_direct().unwrap());
|
||||
}
|
||||
|
||||
let logprobs = raw.new_logprobs.map(|value| value.into_direct().unwrap());
|
||||
|
||||
let finish_reason = finish_reason_from_engine(raw.finish_reason, raw.stop_reason);
|
||||
if let Some(finish_reason) = finish_reason.as_ref() {
|
||||
self.request_metrics.record_finished(received_at, finish_reason.clone());
|
||||
}
|
||||
|
||||
let output = GenerateOutput {
|
||||
request_id: raw.request_id,
|
||||
prompt_info: self.pending_prompt_info.take(),
|
||||
token_ids: raw.new_token_ids,
|
||||
logprobs,
|
||||
finish_reason,
|
||||
kv_transfer_params: raw.kv_transfer_params,
|
||||
};
|
||||
|
||||
Poll::Ready(Some(Ok(output)))
|
||||
}
|
||||
}
|
||||
|
||||
impl FusedStream for GenerateOutputStream {
|
||||
fn is_terminated(&self) -> bool {
|
||||
self.raw_stream.is_terminated()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for GenerateOutputStream {
|
||||
fn drop(&mut self) {
|
||||
if self.raw_stream.is_terminated() {
|
||||
// Already terminated cleanly, no need to record abort metrics.
|
||||
return;
|
||||
}
|
||||
|
||||
// If the user or the upper layer drops a live generate stream,
|
||||
// `EngineCoreOutputStream::Drop` will trigger an engine-side abort. Record the
|
||||
// matching terminal request metrics here so frontend-driven aborts are still
|
||||
// visible as `finished_reason=...` instead of disappearing from observability
|
||||
// entirely.
|
||||
let finish_reason = match AbortCause::current() {
|
||||
AbortCause::DroppedStream => FinishReason::Abort,
|
||||
AbortCause::StopStringMatched => FinishReason::Stop(None),
|
||||
};
|
||||
|
||||
self.request_metrics
|
||||
.record_finished(current_unix_timestamp_secs(), finish_reason);
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::manual_async_fn, reason = "specify `Send` bound")]
|
||||
#[easy_ext::ext(GenerateOutputStreamExt)]
|
||||
impl<T: Stream<Item = Result<GenerateOutput>> + Send> T {
|
||||
/// Collect the raw generate stream to completion and return the final token
|
||||
/// output.
|
||||
pub fn collect_output(self) -> impl Future<Output = Result<CollectedGenerateOutput>> + Send {
|
||||
async move {
|
||||
let stream = self;
|
||||
pin_mut!(stream);
|
||||
let mut prompt_token_ids = None;
|
||||
let mut prompt_logprobs = None;
|
||||
let mut collected: Option<CollectedGenerateOutput> = None;
|
||||
|
||||
while let Some(output) = stream.next().await.transpose()? {
|
||||
if let Some(info) = output.prompt_info {
|
||||
if prompt_token_ids.is_none() {
|
||||
prompt_token_ids = Some(info.prompt_token_ids.to_vec());
|
||||
}
|
||||
if prompt_logprobs.is_none() {
|
||||
prompt_logprobs = info.prompt_logprobs;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(existing) = collected.as_mut() {
|
||||
existing.token_ids.extend(output.token_ids);
|
||||
if let Some(step_logprobs) = output.logprobs {
|
||||
if let Some(collected_logprobs) = existing.logprobs.as_mut() {
|
||||
collected_logprobs.positions.extend(step_logprobs.positions);
|
||||
} else {
|
||||
existing.logprobs = Some(step_logprobs);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
collected = Some(CollectedGenerateOutput {
|
||||
request_id: output.request_id,
|
||||
prompt_token_ids: prompt_token_ids.take().unwrap_or_default(),
|
||||
prompt_logprobs: prompt_logprobs.take(),
|
||||
token_ids: output.token_ids,
|
||||
logprobs: output.logprobs,
|
||||
finish_reason: FinishReason::Error,
|
||||
kv_transfer_params: None,
|
||||
});
|
||||
}
|
||||
|
||||
if let Some(finish_reason) = output.finish_reason {
|
||||
let mut collected = collected.expect("terminal output must exist");
|
||||
collected.finish_reason = finish_reason;
|
||||
collected.kv_transfer_params = output.kv_transfer_params;
|
||||
return Ok(collected);
|
||||
}
|
||||
}
|
||||
|
||||
unreachable!("generate stream should yield an error instead of closing early")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use uuid::Uuid;
|
||||
use vllm_engine_core_client::protocol::multimodal::MmFeatures;
|
||||
use vllm_engine_core_client::protocol::{EngineCoreRequest, EngineCoreSamplingParams, OpaqueValue};
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
|
||||
/// Tokenized decoder-only generate request accepted by [`crate::Llm`].
|
||||
///
|
||||
/// This is the first-stage Rust subset of the inputs that eventually flow into
|
||||
/// Python `AsyncLLM.generate()`. The boundary is intentionally above
|
||||
/// [`EngineCoreRequest`], but below higher-level text and multimodal
|
||||
/// preprocessing.
|
||||
///
|
||||
/// Original Python API reference:
|
||||
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/engine/protocol.py#L67-L84>
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct GenerateRequest {
|
||||
/// Unique ID of the request.
|
||||
pub request_id: String,
|
||||
/// Token IDs of the prompt.
|
||||
pub prompt_token_ids: Vec<u32>,
|
||||
/// Sampling parameters forwarded to engine-core.
|
||||
pub sampling_params: EngineCoreSamplingParams,
|
||||
/// Optional multimodal features already prepared by `vllm-chat`.
|
||||
pub mm_features: Option<MmFeatures>,
|
||||
|
||||
// Fields below are currently likely unused by callers.
|
||||
pub arrival_time: Option<f64>,
|
||||
pub cache_salt: Option<String>,
|
||||
pub trace_headers: Option<BTreeMap<String, String>>,
|
||||
pub priority: i32,
|
||||
pub data_parallel_rank: Option<u32>,
|
||||
pub reasoning_ended: Option<bool>,
|
||||
pub lora_request: Option<OpaqueValue>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct PreparedGenerateRequest {
|
||||
pub engine_request: EngineCoreRequest,
|
||||
}
|
||||
|
||||
impl GenerateRequest {
|
||||
/// Validate and lower this request into the raw engine-core request format.
|
||||
pub(crate) fn prepare(self, randomize_request_id: bool) -> Result<PreparedGenerateRequest> {
|
||||
if self.prompt_token_ids.is_empty() {
|
||||
return Err(Error::EmptyPromptTokenIds {
|
||||
request_id: self.request_id,
|
||||
});
|
||||
}
|
||||
let GenerateRequest {
|
||||
request_id,
|
||||
prompt_token_ids,
|
||||
sampling_params,
|
||||
mm_features,
|
||||
arrival_time,
|
||||
cache_salt,
|
||||
trace_headers,
|
||||
priority,
|
||||
data_parallel_rank,
|
||||
reasoning_ended,
|
||||
lora_request,
|
||||
} = self;
|
||||
|
||||
let external_request_id = request_id;
|
||||
let engine_request_id = if randomize_request_id {
|
||||
let random_suffix = Uuid::new_v4().simple().to_string();
|
||||
format!("{external_request_id}-{}", &random_suffix[..8])
|
||||
} else {
|
||||
external_request_id.clone()
|
||||
};
|
||||
|
||||
Ok(PreparedGenerateRequest {
|
||||
engine_request: EngineCoreRequest {
|
||||
request_id: engine_request_id,
|
||||
prompt_token_ids: Some(prompt_token_ids),
|
||||
mm_features,
|
||||
sampling_params: Some(sampling_params),
|
||||
pooling_params: None,
|
||||
arrival_time: arrival_time.unwrap_or_else(current_unix_timestamp_secs),
|
||||
lora_request,
|
||||
cache_salt,
|
||||
data_parallel_rank,
|
||||
prompt_embeds: None,
|
||||
prompt_is_token_ids: None,
|
||||
client_index: 0,
|
||||
current_wave: 0,
|
||||
priority,
|
||||
trace_headers,
|
||||
resumable: false,
|
||||
external_req_id: Some(external_request_id),
|
||||
reasoning_ended,
|
||||
reasoning_parser_kwargs: None,
|
||||
abort_immediately: false,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl PreparedGenerateRequest {
|
||||
/// Return the original prompt token IDs copied into the raw engine request.
|
||||
pub fn prompt_token_ids(&self) -> &[u32] {
|
||||
self.engine_request
|
||||
.prompt_token_ids
|
||||
.as_ref()
|
||||
.expect("prepared request must have prompt token ids")
|
||||
}
|
||||
}
|
||||
|
||||
fn current_unix_timestamp_secs() -> f64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.expect("system clock is before unix epoch")
|
||||
.as_secs_f64()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use vllm_engine_core_client::protocol::EngineCoreSamplingParams;
|
||||
|
||||
use super::GenerateRequest;
|
||||
use crate::error::Error;
|
||||
|
||||
fn sample_request() -> GenerateRequest {
|
||||
GenerateRequest {
|
||||
request_id: "req-1".to_string(),
|
||||
prompt_token_ids: vec![11, 22, 33],
|
||||
sampling_params: EngineCoreSamplingParams::for_test(),
|
||||
mm_features: None,
|
||||
arrival_time: Some(42.5),
|
||||
cache_salt: Some("salt".to_string()),
|
||||
trace_headers: Some(BTreeMap::from([(
|
||||
"x-trace-id".to_string(),
|
||||
"abc".to_string(),
|
||||
)])),
|
||||
priority: 3,
|
||||
data_parallel_rank: Some(2),
|
||||
reasoning_ended: Some(true),
|
||||
lora_request: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_builds_engine_core_request() {
|
||||
let prepared = sample_request().prepare(true).unwrap();
|
||||
|
||||
assert_eq!(prepared.prompt_token_ids(), &[11, 22, 33]);
|
||||
|
||||
let request = prepared.engine_request;
|
||||
assert_eq!(request.external_req_id.as_deref(), Some("req-1"));
|
||||
assert!(request.request_id.starts_with("req-1-"));
|
||||
assert_ne!(request.request_id, "req-1");
|
||||
assert_eq!(request.prompt_token_ids.as_deref(), Some(&[11, 22, 33][..]));
|
||||
assert_eq!(request.arrival_time, 42.5);
|
||||
assert_eq!(request.cache_salt.as_deref(), Some("salt"));
|
||||
assert_eq!(request.data_parallel_rank, Some(2));
|
||||
assert_eq!(
|
||||
request.trace_headers,
|
||||
Some(BTreeMap::from([(
|
||||
"x-trace-id".to_string(),
|
||||
"abc".to_string(),
|
||||
)]))
|
||||
);
|
||||
assert_eq!(request.reasoning_ended, Some(true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_rejects_empty_prompt_tokens() {
|
||||
let mut request = sample_request();
|
||||
request.prompt_token_ids.clear();
|
||||
|
||||
let error = request.prepare(true).unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
Error::EmptyPromptTokenIds { request_id } if request_id == "req-1"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_can_preserve_external_request_id() {
|
||||
let prepared = sample_request().prepare(false).unwrap();
|
||||
|
||||
let request = prepared.engine_request;
|
||||
assert_eq!(request.external_req_id.as_deref(), Some("req-1"));
|
||||
assert_eq!(request.request_id, "req-1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_forwards_multimodal_features() {
|
||||
let mut request = sample_request();
|
||||
request.mm_features = Some(Vec::new());
|
||||
|
||||
let prepared = request.prepare(false).unwrap();
|
||||
|
||||
assert_eq!(prepared.engine_request.mm_features, Some(Vec::new()));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,391 @@
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use vllm_engine_core_client::protocol::stats::PrefillStats;
|
||||
use vllm_engine_core_client::protocol::{EngineCoreEvent, EngineCoreEventType, EngineCoreOutput};
|
||||
use vllm_metrics::{
|
||||
EngineLabels, FinishedReasonLabels, METRICS, PromptTokenSourceLabels, RequestMetrics,
|
||||
};
|
||||
|
||||
use crate::FinishReason;
|
||||
|
||||
fn metrics() -> &'static RequestMetrics {
|
||||
&METRICS.request
|
||||
}
|
||||
|
||||
const PROMPT_TOKEN_SOURCE_LOCAL_COMPUTE: &str = "local_compute";
|
||||
const PROMPT_TOKEN_SOURCE_LOCAL_CACHE_HIT: &str = "local_cache_hit";
|
||||
const PROMPT_TOKEN_SOURCE_EXTERNAL_KV_TRANSFER: &str = "external_kv_transfer";
|
||||
|
||||
/// Request-scoped metrics state tracked across streamed engine-core updates.
|
||||
///
|
||||
/// This is the Rust-side counterpart of the Python frontend's request-lifecycle
|
||||
/// bookkeeping, centered on `RequestStateStats` and the per-output/per-finished
|
||||
/// update flow.
|
||||
///
|
||||
/// Original Python definitions:
|
||||
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/metrics/stats.py#L200-L237>
|
||||
///
|
||||
/// Original Python update flow:
|
||||
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/engine/output_processor.py#L600-L677>
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct RequestMetricsTracker {
|
||||
model_name: String,
|
||||
arrival_time: f64,
|
||||
prompt_len: u32,
|
||||
max_tokens_param: Option<u32>,
|
||||
n_param: u32,
|
||||
is_prefilling: bool,
|
||||
queued_ts: f64,
|
||||
scheduled_ts: f64,
|
||||
first_token_ts: f64,
|
||||
last_token_ts: f64,
|
||||
first_token_latency: f64,
|
||||
num_generation_tokens: u32,
|
||||
latest_num_cached_tokens: u32,
|
||||
last_seen_engine_index: u32,
|
||||
}
|
||||
|
||||
impl RequestMetricsTracker {
|
||||
/// Create the per-request tracker from the normalized `llm`-layer request
|
||||
/// context.
|
||||
pub(crate) fn new(
|
||||
model_name: String,
|
||||
arrival_time: f64,
|
||||
prompt_len: u32,
|
||||
max_tokens_param: Option<u32>,
|
||||
n_param: u32,
|
||||
) -> Self {
|
||||
Self {
|
||||
model_name,
|
||||
arrival_time,
|
||||
prompt_len,
|
||||
max_tokens_param,
|
||||
n_param,
|
||||
is_prefilling: true,
|
||||
queued_ts: 0.0,
|
||||
scheduled_ts: 0.0,
|
||||
first_token_ts: 0.0,
|
||||
last_token_ts: 0.0,
|
||||
first_token_latency: 0.0,
|
||||
num_generation_tokens: 0,
|
||||
latest_num_cached_tokens: 0,
|
||||
last_seen_engine_index: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Update request-lifecycle state from one engine-core output item.
|
||||
///
|
||||
/// Original Python stats logic:
|
||||
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/metrics/stats.py#L331-L384>
|
||||
pub(crate) fn observe_output(
|
||||
&mut self,
|
||||
engine_index: u32,
|
||||
batch_timestamp: f64,
|
||||
received_at: f64,
|
||||
output: &EngineCoreOutput,
|
||||
) {
|
||||
self.last_seen_engine_index = engine_index;
|
||||
if let Some(prefill_stats) = &output.prefill_stats {
|
||||
self.latest_num_cached_tokens = prefill_stats.num_cached_tokens;
|
||||
}
|
||||
self.num_generation_tokens += output.new_token_ids.len() as u32;
|
||||
metrics()
|
||||
.generation_tokens
|
||||
.get_or_create(&engine_labels(&self.model_name, engine_index))
|
||||
.inc_by(output.new_token_ids.len() as u64);
|
||||
|
||||
if let Some(events) = &output.events {
|
||||
self.observe_events(engine_index, events);
|
||||
}
|
||||
|
||||
if self.is_prefilling {
|
||||
if let Some(prefill_stats) = &output.prefill_stats {
|
||||
record_prompt_tokens(&self.model_name, engine_index, prefill_stats);
|
||||
}
|
||||
self.first_token_latency = received_at - self.arrival_time;
|
||||
observe_time_to_first_token_seconds(
|
||||
&self.model_name,
|
||||
engine_index,
|
||||
self.first_token_latency,
|
||||
);
|
||||
self.first_token_ts = batch_timestamp;
|
||||
self.is_prefilling = false;
|
||||
} else if self.last_token_ts > 0.0 {
|
||||
observe_inter_token_latency_seconds(
|
||||
&self.model_name,
|
||||
engine_index,
|
||||
batch_timestamp - self.last_token_ts,
|
||||
);
|
||||
}
|
||||
|
||||
self.last_token_ts = batch_timestamp;
|
||||
}
|
||||
|
||||
/// Emit the terminal request metrics once a finished output has been
|
||||
/// observed.
|
||||
///
|
||||
/// Original Python finished-request stats:
|
||||
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/metrics/stats.py#L222-L237>
|
||||
pub(crate) fn record_finished(&self, received_at: f64, finish_reason: FinishReason) {
|
||||
let labels = engine_labels(&self.model_name, self.last_seen_engine_index);
|
||||
let prefill_kv_computed_tokens =
|
||||
self.prompt_len.saturating_sub(self.latest_num_cached_tokens);
|
||||
let e2e_latency_seconds = received_at - self.arrival_time;
|
||||
let queue_time_seconds = diff_or_zero(self.scheduled_ts, self.queued_ts);
|
||||
let prefill_time_seconds = diff_or_zero(self.first_token_ts, self.scheduled_ts);
|
||||
let decode_time_seconds = diff_or_zero(self.last_token_ts, self.first_token_ts);
|
||||
let inference_time_seconds = diff_or_zero(self.last_token_ts, self.scheduled_ts);
|
||||
let time_per_output_token_seconds = if self.num_generation_tokens > 1 {
|
||||
diff_or_zero(self.last_token_ts, self.first_token_ts)
|
||||
/ (self.num_generation_tokens - 1) as f64
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
record_request_success(&self.model_name, self.last_seen_engine_index, finish_reason);
|
||||
metrics()
|
||||
.request_prompt_tokens
|
||||
.get_or_create(&labels)
|
||||
.observe(self.prompt_len as f64);
|
||||
metrics()
|
||||
.request_generation_tokens
|
||||
.get_or_create(&labels)
|
||||
.observe(self.num_generation_tokens as f64);
|
||||
metrics()
|
||||
.request_max_num_generation_tokens
|
||||
.get_or_create(&labels)
|
||||
.observe(self.num_generation_tokens as f64);
|
||||
if let Some(max_tokens_param) = self.max_tokens_param {
|
||||
metrics()
|
||||
.request_params_max_tokens
|
||||
.get_or_create(&labels)
|
||||
.observe(max_tokens_param as f64);
|
||||
}
|
||||
metrics().request_params_n.get_or_create(&labels).observe(self.n_param as f64);
|
||||
metrics()
|
||||
.request_prefill_kv_computed_tokens
|
||||
.get_or_create(&labels)
|
||||
.observe(prefill_kv_computed_tokens as f64);
|
||||
metrics()
|
||||
.e2e_request_latency_seconds
|
||||
.get_or_create(&labels)
|
||||
.observe(e2e_latency_seconds);
|
||||
metrics()
|
||||
.request_queue_time_seconds
|
||||
.get_or_create(&labels)
|
||||
.observe(queue_time_seconds);
|
||||
metrics()
|
||||
.request_prefill_time_seconds
|
||||
.get_or_create(&labels)
|
||||
.observe(prefill_time_seconds);
|
||||
metrics()
|
||||
.request_decode_time_seconds
|
||||
.get_or_create(&labels)
|
||||
.observe(decode_time_seconds);
|
||||
metrics()
|
||||
.request_inference_time_seconds
|
||||
.get_or_create(&labels)
|
||||
.observe(inference_time_seconds);
|
||||
metrics()
|
||||
.request_time_per_output_token_seconds
|
||||
.get_or_create(&labels)
|
||||
.observe(time_per_output_token_seconds);
|
||||
}
|
||||
|
||||
fn observe_events(&mut self, engine_index: u32, events: &[EngineCoreEvent]) {
|
||||
for event in events {
|
||||
match event.r#type {
|
||||
EngineCoreEventType::Queued => {
|
||||
self.queued_ts = event.timestamp;
|
||||
}
|
||||
EngineCoreEventType::Scheduled => {
|
||||
if self.scheduled_ts == 0.0 {
|
||||
self.scheduled_ts = event.timestamp;
|
||||
}
|
||||
}
|
||||
EngineCoreEventType::Preempted => {
|
||||
metrics()
|
||||
.num_preemptions
|
||||
.get_or_create(&engine_labels(&self.model_name, engine_index))
|
||||
.inc();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn engine_labels(model_name: &str, engine: u32) -> EngineLabels {
|
||||
EngineLabels {
|
||||
model_name: model_name.to_string(),
|
||||
engine,
|
||||
}
|
||||
}
|
||||
|
||||
fn observe_time_to_first_token_seconds(model_name: &str, engine: u32, seconds: f64) {
|
||||
metrics()
|
||||
.time_to_first_token_seconds
|
||||
.get_or_create(&engine_labels(model_name, engine))
|
||||
.observe(seconds);
|
||||
}
|
||||
|
||||
fn observe_inter_token_latency_seconds(model_name: &str, engine: u32, seconds: f64) {
|
||||
metrics()
|
||||
.inter_token_latency_seconds
|
||||
.get_or_create(&engine_labels(model_name, engine))
|
||||
.observe(seconds);
|
||||
}
|
||||
|
||||
fn record_request_success(model_name: &str, engine: u32, finish_reason: FinishReason) {
|
||||
metrics()
|
||||
.request_success
|
||||
.get_or_create(&FinishedReasonLabels {
|
||||
model_name: model_name.to_string(),
|
||||
engine,
|
||||
finished_reason: finish_reason.as_str(),
|
||||
})
|
||||
.inc();
|
||||
}
|
||||
|
||||
fn prompt_token_source_labels(
|
||||
model_name: &str,
|
||||
engine: u32,
|
||||
source: &'static str,
|
||||
) -> PromptTokenSourceLabels {
|
||||
PromptTokenSourceLabels {
|
||||
model_name: model_name.to_string(),
|
||||
engine,
|
||||
source,
|
||||
}
|
||||
}
|
||||
|
||||
fn record_prompt_tokens(model_name: &str, engine: u32, prefill_stats: &PrefillStats) {
|
||||
let computed = prefill_stats.num_computed_tokens as u64;
|
||||
let local_cache_hit = prefill_stats.num_local_cached_tokens as u64;
|
||||
let external_kv_transfer = prefill_stats.num_external_cached_tokens as u64;
|
||||
|
||||
metrics()
|
||||
.prompt_tokens
|
||||
.get_or_create(&engine_labels(model_name, engine))
|
||||
.inc_by(prefill_stats.num_prompt_tokens as u64);
|
||||
metrics()
|
||||
.prompt_tokens_by_source
|
||||
.get_or_create(&prompt_token_source_labels(
|
||||
model_name,
|
||||
engine,
|
||||
PROMPT_TOKEN_SOURCE_LOCAL_COMPUTE,
|
||||
))
|
||||
.inc_by(computed);
|
||||
metrics()
|
||||
.prompt_tokens_by_source
|
||||
.get_or_create(&prompt_token_source_labels(
|
||||
model_name,
|
||||
engine,
|
||||
PROMPT_TOKEN_SOURCE_LOCAL_CACHE_HIT,
|
||||
))
|
||||
.inc_by(local_cache_hit);
|
||||
metrics()
|
||||
.prompt_tokens_by_source
|
||||
.get_or_create(&prompt_token_source_labels(
|
||||
model_name,
|
||||
engine,
|
||||
PROMPT_TOKEN_SOURCE_EXTERNAL_KV_TRANSFER,
|
||||
))
|
||||
.inc_by(external_kv_transfer);
|
||||
metrics()
|
||||
.prompt_tokens_cached
|
||||
.get_or_create(&engine_labels(model_name, engine))
|
||||
.inc_by(prefill_stats.num_cached_tokens as u64);
|
||||
}
|
||||
|
||||
fn diff_or_zero(end: f64, start: f64) -> f64 {
|
||||
if end > 0.0 && start > 0.0 && end >= start {
|
||||
end - start
|
||||
} else {
|
||||
0.0
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the current wall-clock time in seconds since the Unix epoch.
|
||||
///
|
||||
/// This is used for frontend-side latency measurements such as TTFT and E2E,
|
||||
/// matching the Python frontend's use of wall-clock request arrival/iteration
|
||||
/// timestamps rather than engine-core's monotonic scheduler timestamps.
|
||||
///
|
||||
/// Original Python request timestamp source:
|
||||
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/metrics/stats.py#L206-L216>
|
||||
pub(crate) fn current_unix_timestamp_secs() -> f64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.expect("system clock is before unix epoch")
|
||||
.as_secs_f64()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use vllm_engine_core_client::protocol::stats::PrefillStats;
|
||||
use vllm_engine_core_client::protocol::{EngineCoreEvent, EngineCoreEventType};
|
||||
|
||||
use super::{RequestMetricsTracker, diff_or_zero};
|
||||
|
||||
#[test]
|
||||
fn tracker_updates_timing_state_across_prefill_decode_and_finish() {
|
||||
let mut tracker = RequestMetricsTracker::new("model".to_string(), 100.0, 64, Some(128), 1);
|
||||
|
||||
tracker.observe_output(
|
||||
2,
|
||||
10.0,
|
||||
100.2,
|
||||
&vllm_engine_core_client::protocol::EngineCoreOutput {
|
||||
request_id: "req-1".to_string(),
|
||||
new_token_ids: vec![1],
|
||||
finish_reason: None,
|
||||
events: Some(vec![
|
||||
EngineCoreEvent {
|
||||
r#type: EngineCoreEventType::Queued,
|
||||
timestamp: 8.0,
|
||||
},
|
||||
EngineCoreEvent {
|
||||
r#type: EngineCoreEventType::Scheduled,
|
||||
timestamp: 9.0,
|
||||
},
|
||||
]),
|
||||
prefill_stats: Some(PrefillStats {
|
||||
num_prompt_tokens: 64,
|
||||
num_computed_tokens: 60,
|
||||
num_cached_tokens: 4,
|
||||
num_local_cached_tokens: 4,
|
||||
num_external_cached_tokens: 0,
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
tracker.observe_output(
|
||||
2,
|
||||
11.5,
|
||||
100.4,
|
||||
&vllm_engine_core_client::protocol::EngineCoreOutput {
|
||||
request_id: "req-1".to_string(),
|
||||
new_token_ids: vec![2, 3],
|
||||
finish_reason: None,
|
||||
events: Some(vec![EngineCoreEvent {
|
||||
r#type: EngineCoreEventType::Preempted,
|
||||
timestamp: 10.5,
|
||||
}]),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
assert!(!tracker.is_prefilling);
|
||||
assert_eq!(tracker.last_seen_engine_index, 2);
|
||||
assert_eq!(tracker.num_generation_tokens, 3);
|
||||
assert_eq!(tracker.queued_ts, 8.0);
|
||||
assert_eq!(tracker.scheduled_ts, 9.0);
|
||||
assert_eq!(tracker.first_token_ts, 10.0);
|
||||
assert_eq!(tracker.last_token_ts, 11.5);
|
||||
assert!((tracker.first_token_latency - 0.2).abs() < 1e-9);
|
||||
assert_eq!(
|
||||
diff_or_zero(tracker.last_token_ts, tracker.first_token_ts),
|
||||
1.5
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,749 @@
|
||||
use std::collections::BTreeSet;
|
||||
use std::sync::Once;
|
||||
use std::time::Duration;
|
||||
|
||||
use futures::StreamExt as _;
|
||||
use tokio::time::timeout;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use uuid::Uuid;
|
||||
use vllm_engine_core_client::protocol::logprobs::{
|
||||
Logprobs, MaybeWireLogprobs, PositionLogprobs, TokenLogprob,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::stats::PrefillStats;
|
||||
use vllm_engine_core_client::protocol::{
|
||||
EngineCoreEvent, EngineCoreEventType, EngineCoreFinishReason, EngineCoreOutput,
|
||||
EngineCoreOutputs, EngineCoreRequest, EngineCoreSamplingParams,
|
||||
};
|
||||
use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
|
||||
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig};
|
||||
use vllm_llm::{
|
||||
Error, FinishReason, GenerateOutputStreamExt as _, GeneratePromptInfo, GenerateRequest, Llm,
|
||||
};
|
||||
use vllm_metrics::METRICS;
|
||||
use zeromq::prelude::{SocketRecv, SocketSend};
|
||||
use zeromq::{DealerSocket, PushSocket, ZmqMessage};
|
||||
|
||||
static TRACING: Once = Once::new();
|
||||
|
||||
fn request_output(
|
||||
request_id: &str,
|
||||
new_token_ids: Vec<u32>,
|
||||
finish_reason: Option<EngineCoreFinishReason>,
|
||||
) -> EngineCoreOutput {
|
||||
request_output_with_events(request_id, new_token_ids, finish_reason, None)
|
||||
}
|
||||
|
||||
fn request_output_with_events(
|
||||
request_id: &str,
|
||||
new_token_ids: Vec<u32>,
|
||||
finish_reason: Option<EngineCoreFinishReason>,
|
||||
events: Option<Vec<EngineCoreEvent>>,
|
||||
) -> EngineCoreOutput {
|
||||
EngineCoreOutput {
|
||||
request_id: request_id.to_string(),
|
||||
new_token_ids,
|
||||
new_logprobs: None,
|
||||
new_prompt_logprobs_tensors: None,
|
||||
pooling_output: None,
|
||||
finish_reason,
|
||||
stop_reason: None,
|
||||
events,
|
||||
kv_transfer_params: None,
|
||||
trace_headers: None,
|
||||
prefill_stats: None,
|
||||
routed_experts: None,
|
||||
num_nans_in_logits: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn request_output_with_logprobs(
|
||||
request_id: &str,
|
||||
new_token_ids: Vec<u32>,
|
||||
finish_reason: Option<EngineCoreFinishReason>,
|
||||
new_logprobs: Option<Logprobs>,
|
||||
prompt_logprobs: Option<Logprobs>,
|
||||
) -> EngineCoreOutput {
|
||||
EngineCoreOutput {
|
||||
request_id: request_id.to_string(),
|
||||
new_token_ids,
|
||||
new_logprobs: new_logprobs.map(MaybeWireLogprobs::Direct),
|
||||
new_prompt_logprobs_tensors: prompt_logprobs.map(MaybeWireLogprobs::Direct),
|
||||
pooling_output: None,
|
||||
finish_reason,
|
||||
stop_reason: None,
|
||||
events: None,
|
||||
kv_transfer_params: None,
|
||||
trace_headers: None,
|
||||
prefill_stats: None,
|
||||
routed_experts: None,
|
||||
num_nans_in_logits: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn request_output_with_logprobs_and_kv(
|
||||
request_id: &str,
|
||||
new_token_ids: Vec<u32>,
|
||||
finish_reason: Option<EngineCoreFinishReason>,
|
||||
new_logprobs: Option<Logprobs>,
|
||||
prompt_logprobs: Option<Logprobs>,
|
||||
kv_transfer_params: Option<serde_json::Value>,
|
||||
) -> EngineCoreOutput {
|
||||
EngineCoreOutput {
|
||||
request_id: request_id.to_string(),
|
||||
new_token_ids,
|
||||
new_logprobs: new_logprobs.map(MaybeWireLogprobs::Direct),
|
||||
new_prompt_logprobs_tensors: prompt_logprobs.map(MaybeWireLogprobs::Direct),
|
||||
pooling_output: None,
|
||||
finish_reason,
|
||||
stop_reason: None,
|
||||
events: None,
|
||||
kv_transfer_params,
|
||||
trace_headers: None,
|
||||
prefill_stats: None,
|
||||
routed_experts: None,
|
||||
num_nans_in_logits: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn logprobs_for_position(
|
||||
sampled_token_id: u32,
|
||||
sampled_logprob: f32,
|
||||
sampled_rank: u32,
|
||||
top_token_id: u32,
|
||||
top_logprob: f32,
|
||||
) -> Logprobs {
|
||||
Logprobs {
|
||||
positions: vec![PositionLogprobs {
|
||||
entries: vec![
|
||||
TokenLogprob {
|
||||
token_id: sampled_token_id,
|
||||
logprob: sampled_logprob,
|
||||
rank: sampled_rank,
|
||||
},
|
||||
TokenLogprob {
|
||||
token_id: top_token_id,
|
||||
logprob: top_logprob,
|
||||
rank: 1,
|
||||
},
|
||||
],
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
fn prompt_logprobs() -> Logprobs {
|
||||
Logprobs {
|
||||
positions: vec![
|
||||
PositionLogprobs {
|
||||
entries: vec![
|
||||
TokenLogprob {
|
||||
token_id: 11,
|
||||
logprob: -0.1,
|
||||
rank: 2,
|
||||
},
|
||||
TokenLogprob {
|
||||
token_id: 7,
|
||||
logprob: -0.05,
|
||||
rank: 1,
|
||||
},
|
||||
],
|
||||
},
|
||||
PositionLogprobs {
|
||||
entries: vec![
|
||||
TokenLogprob {
|
||||
token_id: 22,
|
||||
logprob: -0.2,
|
||||
rank: 3,
|
||||
},
|
||||
TokenLogprob {
|
||||
token_id: 8,
|
||||
logprob: -0.1,
|
||||
rank: 1,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_generate_request(request_id: &str, max_tokens: u32) -> GenerateRequest {
|
||||
GenerateRequest {
|
||||
request_id: request_id.to_string(),
|
||||
prompt_token_ids: vec![11, 22],
|
||||
sampling_params: EngineCoreSamplingParams {
|
||||
max_tokens,
|
||||
..EngineCoreSamplingParams::for_test()
|
||||
},
|
||||
mm_features: None,
|
||||
arrival_time: Some(42.5),
|
||||
cache_salt: None,
|
||||
trace_headers: None,
|
||||
priority: 0,
|
||||
data_parallel_rank: None,
|
||||
reasoning_ended: None,
|
||||
lora_request: None,
|
||||
}
|
||||
}
|
||||
|
||||
async fn send_outputs(push: &mut PushSocket, outputs: EngineCoreOutputs) {
|
||||
push.send(ZmqMessage::from(rmp_serde::to_vec_named(&outputs).unwrap()))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn recv_engine_message(dealer: &mut DealerSocket) -> Vec<bytes::Bytes> {
|
||||
dealer.recv().await.unwrap().into_vec()
|
||||
}
|
||||
|
||||
async fn connect_async_llm_with_ipc(
|
||||
handshake_address: String,
|
||||
client_index: u32,
|
||||
model_name: &str,
|
||||
ipc: &IpcNamespace,
|
||||
) -> Llm {
|
||||
let client = EngineCoreClient::connect(
|
||||
EngineCoreClientConfig::new_single(handshake_address)
|
||||
.with_model_name(model_name)
|
||||
.with_client_index(client_index)
|
||||
.with_local_input_output_addresses(
|
||||
Some(ipc.input_endpoint()),
|
||||
Some(ipc.output_endpoint()),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
Llm::new(client)
|
||||
}
|
||||
|
||||
fn request_metrics_model_name(prefix: &str) -> String {
|
||||
format!("{prefix}-{}", Uuid::new_v4().simple())
|
||||
}
|
||||
|
||||
fn init_tracing() {
|
||||
TRACING.call_once(|| {
|
||||
let filter = EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| EnvFilter::new("vllm_engine_core_client=debug"));
|
||||
let _ = tracing_subscriber::fmt().with_test_writer().with_env_filter(filter).try_init();
|
||||
});
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn generate_streams_outputs() {
|
||||
init_tracing();
|
||||
let ipc = IpcNamespace::new().unwrap();
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
let engine_id = b"engine-delta".to_vec();
|
||||
|
||||
let (shutdown_tx, engine_task) = spawn_mock_engine_task(
|
||||
handshake_address.clone(),
|
||||
engine_id.clone(),
|
||||
|dealer, push| {
|
||||
Box::pin(async move {
|
||||
let add = recv_engine_message(dealer).await;
|
||||
assert_eq!(add[0].as_ref(), &[0x00]);
|
||||
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
|
||||
assert_eq!(request.external_req_id.as_deref(), Some("req-delta"));
|
||||
assert!(request.request_id.starts_with("req-delta-"));
|
||||
assert_ne!(request.request_id, "req-delta");
|
||||
assert_eq!(request.client_index, 7);
|
||||
assert_eq!(request.prompt_token_ids, Some(vec![11, 22]));
|
||||
|
||||
send_outputs(
|
||||
push,
|
||||
EngineCoreOutputs {
|
||||
outputs: vec![
|
||||
request_output_with_logprobs(
|
||||
&request.request_id,
|
||||
vec![1, 2],
|
||||
None,
|
||||
Some(logprobs_for_position(1, -0.3, 4, 9, -0.1)),
|
||||
Some(prompt_logprobs()),
|
||||
),
|
||||
request_output_with_logprobs(
|
||||
&request.request_id,
|
||||
vec![3],
|
||||
Some(EngineCoreFinishReason::Length),
|
||||
Some(logprobs_for_position(3, -0.4, 5, 10, -0.2)),
|
||||
None,
|
||||
),
|
||||
],
|
||||
finished_requests: Some(BTreeSet::from([request.request_id.clone()])),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
})
|
||||
},
|
||||
);
|
||||
|
||||
let llm = connect_async_llm_with_ipc(handshake_address, 7, "test-model", &ipc).await;
|
||||
let mut stream = llm.generate(sample_generate_request("req-delta", 3)).await.unwrap();
|
||||
let internal_id = stream.request_id().to_string();
|
||||
|
||||
let first = stream.next().await.unwrap().unwrap();
|
||||
assert_eq!(first.request_id, internal_id);
|
||||
assert_eq!(
|
||||
first.prompt_info,
|
||||
Some(GeneratePromptInfo {
|
||||
prompt_token_ids: vec![11, 22].into(),
|
||||
prompt_logprobs: Some(prompt_logprobs()),
|
||||
})
|
||||
);
|
||||
assert_eq!(first.token_ids, vec![1, 2]);
|
||||
assert_eq!(
|
||||
first.logprobs,
|
||||
Some(logprobs_for_position(1, -0.3, 4, 9, -0.1))
|
||||
);
|
||||
assert_eq!(first.finish_reason, None);
|
||||
|
||||
let second = stream.next().await.unwrap().unwrap();
|
||||
assert_eq!(second.prompt_info, None);
|
||||
assert_eq!(second.token_ids, vec![3]);
|
||||
assert_eq!(
|
||||
second.logprobs,
|
||||
Some(logprobs_for_position(3, -0.4, 5, 10, -0.2))
|
||||
);
|
||||
assert_eq!(second.finish_reason, Some(FinishReason::Length));
|
||||
assert!(stream.next().await.is_none());
|
||||
|
||||
let _ = shutdown_tx.send(());
|
||||
engine_task.await.unwrap();
|
||||
llm.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn collect_output_aggregates_raw_tokens_logprobs_and_terminal_metadata() {
|
||||
init_tracing();
|
||||
let ipc = IpcNamespace::new().unwrap();
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
let engine_id = b"engine-collect-output".to_vec();
|
||||
|
||||
let (shutdown_tx, engine_task) = spawn_mock_engine_task(
|
||||
handshake_address.clone(),
|
||||
engine_id.clone(),
|
||||
|dealer, push| {
|
||||
Box::pin(async move {
|
||||
let add = recv_engine_message(dealer).await;
|
||||
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
|
||||
assert_eq!(request.external_req_id.as_deref(), Some("req-collect"));
|
||||
assert!(request.request_id.starts_with("req-collect-"));
|
||||
|
||||
send_outputs(
|
||||
push,
|
||||
EngineCoreOutputs {
|
||||
engine_index: 0,
|
||||
outputs: vec![
|
||||
request_output_with_logprobs(
|
||||
&request.request_id,
|
||||
vec![33],
|
||||
None,
|
||||
Some(logprobs_for_position(33, -0.1, 1, 99, -0.2)),
|
||||
Some(prompt_logprobs()),
|
||||
),
|
||||
request_output_with_logprobs_and_kv(
|
||||
&request.request_id,
|
||||
vec![44],
|
||||
Some(EngineCoreFinishReason::Stop),
|
||||
Some(logprobs_for_position(44, -0.3, 1, 88, -0.4)),
|
||||
None,
|
||||
Some(serde_json::json!({"connector": "x"})),
|
||||
),
|
||||
],
|
||||
scheduler_stats: None,
|
||||
timestamp: 0.0,
|
||||
utility_output: None,
|
||||
finished_requests: None,
|
||||
wave_complete: None,
|
||||
start_wave: None,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
})
|
||||
},
|
||||
);
|
||||
|
||||
let llm = connect_async_llm_with_ipc(handshake_address, 7, "test-model", &ipc).await;
|
||||
let stream = llm.generate(sample_generate_request("req-collect", 4)).await.unwrap();
|
||||
let internal_id = stream.request_id().to_string();
|
||||
let collected = stream.collect_output().await.unwrap();
|
||||
|
||||
let _ = shutdown_tx.send(());
|
||||
engine_task.await.unwrap();
|
||||
|
||||
assert_eq!(collected.request_id, internal_id);
|
||||
assert_eq!(collected.prompt_token_ids, vec![11, 22]);
|
||||
assert_eq!(collected.token_ids, vec![33, 44]);
|
||||
assert_eq!(collected.finish_reason, FinishReason::stop_eos());
|
||||
assert_eq!(collected.prompt_logprobs, Some(prompt_logprobs()));
|
||||
assert_eq!(
|
||||
collected.logprobs.as_ref().map(|lp| lp.positions.len()),
|
||||
Some(2)
|
||||
);
|
||||
assert_eq!(
|
||||
collected.kv_transfer_params,
|
||||
Some(serde_json::json!({"connector": "x"}))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn generate_propagates_unexpected_close_errors() {
|
||||
let ipc = IpcNamespace::new().unwrap();
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
let engine_id = b"engine-close".to_vec();
|
||||
|
||||
let (shutdown_tx, engine_task) = spawn_mock_engine_task(
|
||||
handshake_address.clone(),
|
||||
engine_id.clone(),
|
||||
|dealer, push| {
|
||||
Box::pin(async move {
|
||||
let add = recv_engine_message(dealer).await;
|
||||
assert_eq!(add[0].as_ref(), &[0x00]);
|
||||
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
|
||||
|
||||
send_outputs(
|
||||
push,
|
||||
EngineCoreOutputs {
|
||||
finished_requests: Some(BTreeSet::from([request.request_id])),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
})
|
||||
},
|
||||
);
|
||||
|
||||
let llm = connect_async_llm_with_ipc(handshake_address, 0, "test-model", &ipc).await;
|
||||
let mut stream = llm.generate(sample_generate_request("req-close", 1)).await.unwrap();
|
||||
let internal_id = stream.request_id().to_string();
|
||||
|
||||
let error = stream.next().await.unwrap().unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
Error::EngineCoreClient(vllm_engine_core_client::Error::RequestStreamClosed {
|
||||
request_id
|
||||
}) if request_id == internal_id
|
||||
));
|
||||
assert!(stream.next().await.is_none());
|
||||
|
||||
let _ = shutdown_tx.send(());
|
||||
engine_task.await.unwrap();
|
||||
llm.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn dropping_a_live_generate_stream_triggers_abort() {
|
||||
let ipc = IpcNamespace::new().unwrap();
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
let engine_id = b"engine-drop".to_vec();
|
||||
|
||||
let (shutdown_tx, engine_task) = spawn_mock_engine_task(
|
||||
handshake_address.clone(),
|
||||
engine_id.clone(),
|
||||
|dealer, push| {
|
||||
Box::pin(async move {
|
||||
let add = recv_engine_message(dealer).await;
|
||||
assert_eq!(add[0].as_ref(), &[0x00]);
|
||||
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
|
||||
assert_eq!(request.external_req_id.as_deref(), Some("req-drop"));
|
||||
assert!(request.request_id.starts_with("req-drop-"));
|
||||
|
||||
send_outputs(
|
||||
push,
|
||||
EngineCoreOutputs {
|
||||
outputs: vec![request_output(&request.request_id, vec![99], None)],
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
let abort =
|
||||
timeout(Duration::from_secs(1), recv_engine_message(dealer)).await.unwrap();
|
||||
assert_eq!(abort[0].as_ref(), &[0x01]);
|
||||
let aborted_ids: Vec<String> = rmp_serde::from_slice(&abort[1]).unwrap();
|
||||
assert_eq!(aborted_ids, vec![request.request_id]);
|
||||
})
|
||||
},
|
||||
);
|
||||
|
||||
let llm = connect_async_llm_with_ipc(handshake_address, 0, "test-model", &ipc).await;
|
||||
let mut stream = llm.generate(sample_generate_request("req-drop", 4)).await.unwrap();
|
||||
|
||||
let output = stream.next().await.unwrap().unwrap();
|
||||
assert_eq!(output.token_ids, vec![99]);
|
||||
drop(stream);
|
||||
|
||||
let _ = shutdown_tx.send(());
|
||||
engine_task.await.unwrap();
|
||||
llm.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn duplicate_external_request_ids_are_randomized_before_reaching_engine_core_client() {
|
||||
let ipc = IpcNamespace::new().unwrap();
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
let engine_id = b"engine-dup".to_vec();
|
||||
|
||||
let (shutdown_tx, engine_task) = spawn_mock_engine_task(
|
||||
handshake_address.clone(),
|
||||
engine_id.clone(),
|
||||
|dealer, push| {
|
||||
Box::pin(async move {
|
||||
let add_1 = recv_engine_message(dealer).await;
|
||||
assert_eq!(add_1[0].as_ref(), &[0x00]);
|
||||
let request_1: EngineCoreRequest = rmp_serde::from_slice(&add_1[1]).unwrap();
|
||||
assert_eq!(request_1.external_req_id.as_deref(), Some("req-dup"));
|
||||
assert!(request_1.request_id.starts_with("req-dup-"));
|
||||
|
||||
let add_2 = recv_engine_message(dealer).await;
|
||||
assert_eq!(add_2[0].as_ref(), &[0x00]);
|
||||
let request_2: EngineCoreRequest = rmp_serde::from_slice(&add_2[1]).unwrap();
|
||||
assert_eq!(request_2.external_req_id.as_deref(), Some("req-dup"));
|
||||
assert!(request_2.request_id.starts_with("req-dup-"));
|
||||
assert_ne!(request_1.request_id, request_2.request_id);
|
||||
|
||||
send_outputs(
|
||||
push,
|
||||
EngineCoreOutputs {
|
||||
outputs: vec![request_output(
|
||||
&request_1.request_id,
|
||||
vec![],
|
||||
Some(EngineCoreFinishReason::Length),
|
||||
)],
|
||||
finished_requests: Some(BTreeSet::from([request_1.request_id.clone()])),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
send_outputs(
|
||||
push,
|
||||
EngineCoreOutputs {
|
||||
outputs: vec![request_output(
|
||||
&request_2.request_id,
|
||||
vec![],
|
||||
Some(EngineCoreFinishReason::Length),
|
||||
)],
|
||||
finished_requests: Some(BTreeSet::from([request_2.request_id])),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
})
|
||||
},
|
||||
);
|
||||
|
||||
let llm = connect_async_llm_with_ipc(handshake_address, 0, "test-model", &ipc).await;
|
||||
let stream_1 = llm.generate(sample_generate_request("req-dup", 1)).await.unwrap();
|
||||
let stream_2 = llm.generate(sample_generate_request("req-dup", 1)).await.unwrap();
|
||||
let internal_id_1 = stream_1.request_id().to_string();
|
||||
let internal_id_2 = stream_2.request_id().to_string();
|
||||
let collected_1 = stream_1.collect_output().await.unwrap();
|
||||
let collected_2 = stream_2.collect_output().await.unwrap();
|
||||
assert_eq!(collected_1.request_id, internal_id_1);
|
||||
assert_eq!(collected_2.request_id, internal_id_2);
|
||||
let _ = shutdown_tx.send(());
|
||||
engine_task.await.unwrap();
|
||||
llm.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn generate_records_request_metrics_in_prometheus_output() {
|
||||
let ipc = IpcNamespace::new().unwrap();
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
let engine_id = b"engine-metrics".to_vec();
|
||||
let model_name = request_metrics_model_name("metrics-model");
|
||||
|
||||
let (shutdown_tx, engine_task) = spawn_mock_engine_task(
|
||||
handshake_address.clone(),
|
||||
engine_id.clone(),
|
||||
|dealer, push| {
|
||||
Box::pin(async move {
|
||||
let add = recv_engine_message(dealer).await;
|
||||
assert_eq!(add[0].as_ref(), &[0x00]);
|
||||
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
|
||||
|
||||
send_outputs(
|
||||
push,
|
||||
EngineCoreOutputs {
|
||||
engine_index: 4,
|
||||
timestamp: 10.0,
|
||||
outputs: vec![EngineCoreOutput {
|
||||
prefill_stats: Some(PrefillStats {
|
||||
num_prompt_tokens: 2,
|
||||
num_computed_tokens: 2,
|
||||
..Default::default()
|
||||
}),
|
||||
..request_output_with_events(
|
||||
&request.request_id,
|
||||
vec![1],
|
||||
None,
|
||||
Some(vec![
|
||||
EngineCoreEvent {
|
||||
r#type: EngineCoreEventType::Queued,
|
||||
timestamp: 8.0,
|
||||
},
|
||||
EngineCoreEvent {
|
||||
r#type: EngineCoreEventType::Scheduled,
|
||||
timestamp: 9.0,
|
||||
},
|
||||
]),
|
||||
)
|
||||
}],
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
send_outputs(
|
||||
push,
|
||||
EngineCoreOutputs {
|
||||
engine_index: 4,
|
||||
timestamp: 11.5,
|
||||
outputs: vec![request_output_with_events(
|
||||
&request.request_id,
|
||||
vec![2, 3],
|
||||
Some(EngineCoreFinishReason::Length),
|
||||
Some(vec![EngineCoreEvent {
|
||||
r#type: EngineCoreEventType::Preempted,
|
||||
timestamp: 10.5,
|
||||
}]),
|
||||
)],
|
||||
finished_requests: Some(BTreeSet::from([request.request_id])),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
})
|
||||
},
|
||||
);
|
||||
|
||||
let llm = connect_async_llm_with_ipc(handshake_address, 0, &model_name, &ipc).await;
|
||||
let mut request = sample_generate_request("req-metrics", 8);
|
||||
request.arrival_time = None;
|
||||
let mut stream = llm.generate(request).await.unwrap();
|
||||
|
||||
assert_eq!(stream.next().await.unwrap().unwrap().token_ids, vec![1]);
|
||||
let final_output = stream.next().await.unwrap().unwrap();
|
||||
assert_eq!(final_output.token_ids, vec![2, 3]);
|
||||
assert_eq!(final_output.finish_reason, Some(FinishReason::Length));
|
||||
assert!(stream.next().await.is_none());
|
||||
|
||||
let rendered = METRICS.render().unwrap();
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:request_success_total{{model_name=\"{model_name}\",engine=\"4\",finished_reason=\"length\"}} 1"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:prompt_tokens_total{{model_name=\"{model_name}\",engine=\"4\"}} 2"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:prompt_tokens_by_source_total{{model_name=\"{model_name}\",engine=\"4\",source=\"local_compute\"}} 2"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:prompt_tokens_by_source_total{{model_name=\"{model_name}\",engine=\"4\",source=\"local_cache_hit\"}} 0"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:prompt_tokens_by_source_total{{model_name=\"{model_name}\",engine=\"4\",source=\"external_kv_transfer\"}} 0"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:prompt_tokens_cached_total{{model_name=\"{model_name}\",engine=\"4\"}} 0"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:generation_tokens_total{{model_name=\"{model_name}\",engine=\"4\"}} 3"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:num_preemptions_total{{model_name=\"{model_name}\",engine=\"4\"}} 1"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:time_to_first_token_seconds_count{{model_name=\"{model_name}\",engine=\"4\"}} 1"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:inter_token_latency_seconds_count{{model_name=\"{model_name}\",engine=\"4\"}} 1"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:e2e_request_latency_seconds_count{{model_name=\"{model_name}\",engine=\"4\"}} 1"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:request_prompt_tokens_count{{model_name=\"{model_name}\",engine=\"4\"}} 1"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:request_generation_tokens_count{{model_name=\"{model_name}\",engine=\"4\"}} 1"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:request_prefill_kv_computed_tokens_count{{model_name=\"{model_name}\",engine=\"4\"}} 1"
|
||||
)));
|
||||
|
||||
let _ = shutdown_tx.send(());
|
||||
engine_task.await.unwrap();
|
||||
llm.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn dropping_stream_records_abort_terminal_request_metrics() {
|
||||
let ipc = IpcNamespace::new().unwrap();
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
let engine_id = b"engine-metrics-drop".to_vec();
|
||||
let model_name = request_metrics_model_name("metrics-drop-model");
|
||||
|
||||
let (shutdown_tx, engine_task) = spawn_mock_engine_task(
|
||||
handshake_address.clone(),
|
||||
engine_id.clone(),
|
||||
|dealer, push| {
|
||||
Box::pin(async move {
|
||||
let add = recv_engine_message(dealer).await;
|
||||
assert_eq!(add[0].as_ref(), &[0x00]);
|
||||
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
|
||||
assert_eq!(request.external_req_id.as_deref(), Some("req-metrics-drop"));
|
||||
assert!(request.request_id.starts_with("req-metrics-drop-"));
|
||||
|
||||
send_outputs(
|
||||
push,
|
||||
EngineCoreOutputs {
|
||||
engine_index: 5,
|
||||
timestamp: 10.0,
|
||||
outputs: vec![request_output_with_events(
|
||||
&request.request_id,
|
||||
vec![99],
|
||||
None,
|
||||
Some(vec![
|
||||
EngineCoreEvent {
|
||||
r#type: EngineCoreEventType::Queued,
|
||||
timestamp: 8.0,
|
||||
},
|
||||
EngineCoreEvent {
|
||||
r#type: EngineCoreEventType::Scheduled,
|
||||
timestamp: 9.0,
|
||||
},
|
||||
]),
|
||||
)],
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
let abort =
|
||||
timeout(Duration::from_secs(1), recv_engine_message(dealer)).await.unwrap();
|
||||
assert_eq!(abort[0].as_ref(), &[0x01]);
|
||||
let aborted_ids: Vec<String> = rmp_serde::from_slice(&abort[1]).unwrap();
|
||||
assert_eq!(aborted_ids, vec![request.request_id]);
|
||||
})
|
||||
},
|
||||
);
|
||||
|
||||
let llm = connect_async_llm_with_ipc(handshake_address, 0, &model_name, &ipc).await;
|
||||
let mut request = sample_generate_request("req-metrics-drop", 8);
|
||||
request.arrival_time = None;
|
||||
let mut stream = llm.generate(request).await.unwrap();
|
||||
assert_eq!(stream.next().await.unwrap().unwrap().token_ids, vec![99]);
|
||||
drop(stream);
|
||||
|
||||
let _ = shutdown_tx.send(());
|
||||
engine_task.await.unwrap();
|
||||
let rendered = METRICS.render().unwrap();
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:request_success_total{{model_name=\"{model_name}\",engine=\"5\",finished_reason=\"abort\"}} 1"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:e2e_request_latency_seconds_count{{model_name=\"{model_name}\",engine=\"5\"}} 1"
|
||||
)));
|
||||
|
||||
llm.shutdown().await.unwrap();
|
||||
}
|
||||
Reference in New Issue
Block a user