[Rust Frontend] Move code from vllm-frontend-rs (#43283)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Signed-off-by: Eric Curtin <eric.curtin@docker.com> Signed-off-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com> Signed-off-by: Will.hou <1205157517@qq.com> Signed-off-by: Will.hou <willamhou@ceresman.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Eric Curtin <eric.curtin@docker.com> Co-authored-by: Dev-X25874 <283057883+Dev-X25874@users.noreply.github.com> Co-authored-by: Will.hou <1205157517@qq.com> Co-authored-by: Will.hou <willamhou@ceresman.com> Please see https://github.com/Inferact/vllm-frontend-rs for full original commit history.
This commit is contained in:
co-authored by
Nick Hill
Eric Curtin
Dev-X25874
Will.hou
Will.hou
parent
39d5fa96a7
commit
39910f2b25
@@ -0,0 +1,749 @@
|
||||
use std::collections::BTreeSet;
|
||||
use std::sync::Once;
|
||||
use std::time::Duration;
|
||||
|
||||
use futures::StreamExt as _;
|
||||
use tokio::time::timeout;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use uuid::Uuid;
|
||||
use vllm_engine_core_client::protocol::logprobs::{
|
||||
Logprobs, MaybeWireLogprobs, PositionLogprobs, TokenLogprob,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::stats::PrefillStats;
|
||||
use vllm_engine_core_client::protocol::{
|
||||
EngineCoreEvent, EngineCoreEventType, EngineCoreFinishReason, EngineCoreOutput,
|
||||
EngineCoreOutputs, EngineCoreRequest, EngineCoreSamplingParams,
|
||||
};
|
||||
use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
|
||||
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig};
|
||||
use vllm_llm::{
|
||||
Error, FinishReason, GenerateOutputStreamExt as _, GeneratePromptInfo, GenerateRequest, Llm,
|
||||
};
|
||||
use vllm_metrics::METRICS;
|
||||
use zeromq::prelude::{SocketRecv, SocketSend};
|
||||
use zeromq::{DealerSocket, PushSocket, ZmqMessage};
|
||||
|
||||
static TRACING: Once = Once::new();
|
||||
|
||||
fn request_output(
|
||||
request_id: &str,
|
||||
new_token_ids: Vec<u32>,
|
||||
finish_reason: Option<EngineCoreFinishReason>,
|
||||
) -> EngineCoreOutput {
|
||||
request_output_with_events(request_id, new_token_ids, finish_reason, None)
|
||||
}
|
||||
|
||||
fn request_output_with_events(
|
||||
request_id: &str,
|
||||
new_token_ids: Vec<u32>,
|
||||
finish_reason: Option<EngineCoreFinishReason>,
|
||||
events: Option<Vec<EngineCoreEvent>>,
|
||||
) -> EngineCoreOutput {
|
||||
EngineCoreOutput {
|
||||
request_id: request_id.to_string(),
|
||||
new_token_ids,
|
||||
new_logprobs: None,
|
||||
new_prompt_logprobs_tensors: None,
|
||||
pooling_output: None,
|
||||
finish_reason,
|
||||
stop_reason: None,
|
||||
events,
|
||||
kv_transfer_params: None,
|
||||
trace_headers: None,
|
||||
prefill_stats: None,
|
||||
routed_experts: None,
|
||||
num_nans_in_logits: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn request_output_with_logprobs(
|
||||
request_id: &str,
|
||||
new_token_ids: Vec<u32>,
|
||||
finish_reason: Option<EngineCoreFinishReason>,
|
||||
new_logprobs: Option<Logprobs>,
|
||||
prompt_logprobs: Option<Logprobs>,
|
||||
) -> EngineCoreOutput {
|
||||
EngineCoreOutput {
|
||||
request_id: request_id.to_string(),
|
||||
new_token_ids,
|
||||
new_logprobs: new_logprobs.map(MaybeWireLogprobs::Direct),
|
||||
new_prompt_logprobs_tensors: prompt_logprobs.map(MaybeWireLogprobs::Direct),
|
||||
pooling_output: None,
|
||||
finish_reason,
|
||||
stop_reason: None,
|
||||
events: None,
|
||||
kv_transfer_params: None,
|
||||
trace_headers: None,
|
||||
prefill_stats: None,
|
||||
routed_experts: None,
|
||||
num_nans_in_logits: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn request_output_with_logprobs_and_kv(
|
||||
request_id: &str,
|
||||
new_token_ids: Vec<u32>,
|
||||
finish_reason: Option<EngineCoreFinishReason>,
|
||||
new_logprobs: Option<Logprobs>,
|
||||
prompt_logprobs: Option<Logprobs>,
|
||||
kv_transfer_params: Option<serde_json::Value>,
|
||||
) -> EngineCoreOutput {
|
||||
EngineCoreOutput {
|
||||
request_id: request_id.to_string(),
|
||||
new_token_ids,
|
||||
new_logprobs: new_logprobs.map(MaybeWireLogprobs::Direct),
|
||||
new_prompt_logprobs_tensors: prompt_logprobs.map(MaybeWireLogprobs::Direct),
|
||||
pooling_output: None,
|
||||
finish_reason,
|
||||
stop_reason: None,
|
||||
events: None,
|
||||
kv_transfer_params,
|
||||
trace_headers: None,
|
||||
prefill_stats: None,
|
||||
routed_experts: None,
|
||||
num_nans_in_logits: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn logprobs_for_position(
|
||||
sampled_token_id: u32,
|
||||
sampled_logprob: f32,
|
||||
sampled_rank: u32,
|
||||
top_token_id: u32,
|
||||
top_logprob: f32,
|
||||
) -> Logprobs {
|
||||
Logprobs {
|
||||
positions: vec![PositionLogprobs {
|
||||
entries: vec![
|
||||
TokenLogprob {
|
||||
token_id: sampled_token_id,
|
||||
logprob: sampled_logprob,
|
||||
rank: sampled_rank,
|
||||
},
|
||||
TokenLogprob {
|
||||
token_id: top_token_id,
|
||||
logprob: top_logprob,
|
||||
rank: 1,
|
||||
},
|
||||
],
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
fn prompt_logprobs() -> Logprobs {
|
||||
Logprobs {
|
||||
positions: vec![
|
||||
PositionLogprobs {
|
||||
entries: vec![
|
||||
TokenLogprob {
|
||||
token_id: 11,
|
||||
logprob: -0.1,
|
||||
rank: 2,
|
||||
},
|
||||
TokenLogprob {
|
||||
token_id: 7,
|
||||
logprob: -0.05,
|
||||
rank: 1,
|
||||
},
|
||||
],
|
||||
},
|
||||
PositionLogprobs {
|
||||
entries: vec![
|
||||
TokenLogprob {
|
||||
token_id: 22,
|
||||
logprob: -0.2,
|
||||
rank: 3,
|
||||
},
|
||||
TokenLogprob {
|
||||
token_id: 8,
|
||||
logprob: -0.1,
|
||||
rank: 1,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_generate_request(request_id: &str, max_tokens: u32) -> GenerateRequest {
|
||||
GenerateRequest {
|
||||
request_id: request_id.to_string(),
|
||||
prompt_token_ids: vec![11, 22],
|
||||
sampling_params: EngineCoreSamplingParams {
|
||||
max_tokens,
|
||||
..EngineCoreSamplingParams::for_test()
|
||||
},
|
||||
mm_features: None,
|
||||
arrival_time: Some(42.5),
|
||||
cache_salt: None,
|
||||
trace_headers: None,
|
||||
priority: 0,
|
||||
data_parallel_rank: None,
|
||||
reasoning_ended: None,
|
||||
lora_request: None,
|
||||
}
|
||||
}
|
||||
|
||||
async fn send_outputs(push: &mut PushSocket, outputs: EngineCoreOutputs) {
|
||||
push.send(ZmqMessage::from(rmp_serde::to_vec_named(&outputs).unwrap()))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn recv_engine_message(dealer: &mut DealerSocket) -> Vec<bytes::Bytes> {
|
||||
dealer.recv().await.unwrap().into_vec()
|
||||
}
|
||||
|
||||
async fn connect_async_llm_with_ipc(
|
||||
handshake_address: String,
|
||||
client_index: u32,
|
||||
model_name: &str,
|
||||
ipc: &IpcNamespace,
|
||||
) -> Llm {
|
||||
let client = EngineCoreClient::connect(
|
||||
EngineCoreClientConfig::new_single(handshake_address)
|
||||
.with_model_name(model_name)
|
||||
.with_client_index(client_index)
|
||||
.with_local_input_output_addresses(
|
||||
Some(ipc.input_endpoint()),
|
||||
Some(ipc.output_endpoint()),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
Llm::new(client)
|
||||
}
|
||||
|
||||
fn request_metrics_model_name(prefix: &str) -> String {
|
||||
format!("{prefix}-{}", Uuid::new_v4().simple())
|
||||
}
|
||||
|
||||
fn init_tracing() {
|
||||
TRACING.call_once(|| {
|
||||
let filter = EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| EnvFilter::new("vllm_engine_core_client=debug"));
|
||||
let _ = tracing_subscriber::fmt().with_test_writer().with_env_filter(filter).try_init();
|
||||
});
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn generate_streams_outputs() {
|
||||
init_tracing();
|
||||
let ipc = IpcNamespace::new().unwrap();
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
let engine_id = b"engine-delta".to_vec();
|
||||
|
||||
let (shutdown_tx, engine_task) = spawn_mock_engine_task(
|
||||
handshake_address.clone(),
|
||||
engine_id.clone(),
|
||||
|dealer, push| {
|
||||
Box::pin(async move {
|
||||
let add = recv_engine_message(dealer).await;
|
||||
assert_eq!(add[0].as_ref(), &[0x00]);
|
||||
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
|
||||
assert_eq!(request.external_req_id.as_deref(), Some("req-delta"));
|
||||
assert!(request.request_id.starts_with("req-delta-"));
|
||||
assert_ne!(request.request_id, "req-delta");
|
||||
assert_eq!(request.client_index, 7);
|
||||
assert_eq!(request.prompt_token_ids, Some(vec![11, 22]));
|
||||
|
||||
send_outputs(
|
||||
push,
|
||||
EngineCoreOutputs {
|
||||
outputs: vec![
|
||||
request_output_with_logprobs(
|
||||
&request.request_id,
|
||||
vec![1, 2],
|
||||
None,
|
||||
Some(logprobs_for_position(1, -0.3, 4, 9, -0.1)),
|
||||
Some(prompt_logprobs()),
|
||||
),
|
||||
request_output_with_logprobs(
|
||||
&request.request_id,
|
||||
vec![3],
|
||||
Some(EngineCoreFinishReason::Length),
|
||||
Some(logprobs_for_position(3, -0.4, 5, 10, -0.2)),
|
||||
None,
|
||||
),
|
||||
],
|
||||
finished_requests: Some(BTreeSet::from([request.request_id.clone()])),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
})
|
||||
},
|
||||
);
|
||||
|
||||
let llm = connect_async_llm_with_ipc(handshake_address, 7, "test-model", &ipc).await;
|
||||
let mut stream = llm.generate(sample_generate_request("req-delta", 3)).await.unwrap();
|
||||
let internal_id = stream.request_id().to_string();
|
||||
|
||||
let first = stream.next().await.unwrap().unwrap();
|
||||
assert_eq!(first.request_id, internal_id);
|
||||
assert_eq!(
|
||||
first.prompt_info,
|
||||
Some(GeneratePromptInfo {
|
||||
prompt_token_ids: vec![11, 22].into(),
|
||||
prompt_logprobs: Some(prompt_logprobs()),
|
||||
})
|
||||
);
|
||||
assert_eq!(first.token_ids, vec![1, 2]);
|
||||
assert_eq!(
|
||||
first.logprobs,
|
||||
Some(logprobs_for_position(1, -0.3, 4, 9, -0.1))
|
||||
);
|
||||
assert_eq!(first.finish_reason, None);
|
||||
|
||||
let second = stream.next().await.unwrap().unwrap();
|
||||
assert_eq!(second.prompt_info, None);
|
||||
assert_eq!(second.token_ids, vec![3]);
|
||||
assert_eq!(
|
||||
second.logprobs,
|
||||
Some(logprobs_for_position(3, -0.4, 5, 10, -0.2))
|
||||
);
|
||||
assert_eq!(second.finish_reason, Some(FinishReason::Length));
|
||||
assert!(stream.next().await.is_none());
|
||||
|
||||
let _ = shutdown_tx.send(());
|
||||
engine_task.await.unwrap();
|
||||
llm.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn collect_output_aggregates_raw_tokens_logprobs_and_terminal_metadata() {
|
||||
init_tracing();
|
||||
let ipc = IpcNamespace::new().unwrap();
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
let engine_id = b"engine-collect-output".to_vec();
|
||||
|
||||
let (shutdown_tx, engine_task) = spawn_mock_engine_task(
|
||||
handshake_address.clone(),
|
||||
engine_id.clone(),
|
||||
|dealer, push| {
|
||||
Box::pin(async move {
|
||||
let add = recv_engine_message(dealer).await;
|
||||
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
|
||||
assert_eq!(request.external_req_id.as_deref(), Some("req-collect"));
|
||||
assert!(request.request_id.starts_with("req-collect-"));
|
||||
|
||||
send_outputs(
|
||||
push,
|
||||
EngineCoreOutputs {
|
||||
engine_index: 0,
|
||||
outputs: vec![
|
||||
request_output_with_logprobs(
|
||||
&request.request_id,
|
||||
vec![33],
|
||||
None,
|
||||
Some(logprobs_for_position(33, -0.1, 1, 99, -0.2)),
|
||||
Some(prompt_logprobs()),
|
||||
),
|
||||
request_output_with_logprobs_and_kv(
|
||||
&request.request_id,
|
||||
vec![44],
|
||||
Some(EngineCoreFinishReason::Stop),
|
||||
Some(logprobs_for_position(44, -0.3, 1, 88, -0.4)),
|
||||
None,
|
||||
Some(serde_json::json!({"connector": "x"})),
|
||||
),
|
||||
],
|
||||
scheduler_stats: None,
|
||||
timestamp: 0.0,
|
||||
utility_output: None,
|
||||
finished_requests: None,
|
||||
wave_complete: None,
|
||||
start_wave: None,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
})
|
||||
},
|
||||
);
|
||||
|
||||
let llm = connect_async_llm_with_ipc(handshake_address, 7, "test-model", &ipc).await;
|
||||
let stream = llm.generate(sample_generate_request("req-collect", 4)).await.unwrap();
|
||||
let internal_id = stream.request_id().to_string();
|
||||
let collected = stream.collect_output().await.unwrap();
|
||||
|
||||
let _ = shutdown_tx.send(());
|
||||
engine_task.await.unwrap();
|
||||
|
||||
assert_eq!(collected.request_id, internal_id);
|
||||
assert_eq!(collected.prompt_token_ids, vec![11, 22]);
|
||||
assert_eq!(collected.token_ids, vec![33, 44]);
|
||||
assert_eq!(collected.finish_reason, FinishReason::stop_eos());
|
||||
assert_eq!(collected.prompt_logprobs, Some(prompt_logprobs()));
|
||||
assert_eq!(
|
||||
collected.logprobs.as_ref().map(|lp| lp.positions.len()),
|
||||
Some(2)
|
||||
);
|
||||
assert_eq!(
|
||||
collected.kv_transfer_params,
|
||||
Some(serde_json::json!({"connector": "x"}))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn generate_propagates_unexpected_close_errors() {
|
||||
let ipc = IpcNamespace::new().unwrap();
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
let engine_id = b"engine-close".to_vec();
|
||||
|
||||
let (shutdown_tx, engine_task) = spawn_mock_engine_task(
|
||||
handshake_address.clone(),
|
||||
engine_id.clone(),
|
||||
|dealer, push| {
|
||||
Box::pin(async move {
|
||||
let add = recv_engine_message(dealer).await;
|
||||
assert_eq!(add[0].as_ref(), &[0x00]);
|
||||
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
|
||||
|
||||
send_outputs(
|
||||
push,
|
||||
EngineCoreOutputs {
|
||||
finished_requests: Some(BTreeSet::from([request.request_id])),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
})
|
||||
},
|
||||
);
|
||||
|
||||
let llm = connect_async_llm_with_ipc(handshake_address, 0, "test-model", &ipc).await;
|
||||
let mut stream = llm.generate(sample_generate_request("req-close", 1)).await.unwrap();
|
||||
let internal_id = stream.request_id().to_string();
|
||||
|
||||
let error = stream.next().await.unwrap().unwrap_err();
|
||||
assert!(matches!(
|
||||
error,
|
||||
Error::EngineCoreClient(vllm_engine_core_client::Error::RequestStreamClosed {
|
||||
request_id
|
||||
}) if request_id == internal_id
|
||||
));
|
||||
assert!(stream.next().await.is_none());
|
||||
|
||||
let _ = shutdown_tx.send(());
|
||||
engine_task.await.unwrap();
|
||||
llm.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn dropping_a_live_generate_stream_triggers_abort() {
|
||||
let ipc = IpcNamespace::new().unwrap();
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
let engine_id = b"engine-drop".to_vec();
|
||||
|
||||
let (shutdown_tx, engine_task) = spawn_mock_engine_task(
|
||||
handshake_address.clone(),
|
||||
engine_id.clone(),
|
||||
|dealer, push| {
|
||||
Box::pin(async move {
|
||||
let add = recv_engine_message(dealer).await;
|
||||
assert_eq!(add[0].as_ref(), &[0x00]);
|
||||
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
|
||||
assert_eq!(request.external_req_id.as_deref(), Some("req-drop"));
|
||||
assert!(request.request_id.starts_with("req-drop-"));
|
||||
|
||||
send_outputs(
|
||||
push,
|
||||
EngineCoreOutputs {
|
||||
outputs: vec![request_output(&request.request_id, vec![99], None)],
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
let abort =
|
||||
timeout(Duration::from_secs(1), recv_engine_message(dealer)).await.unwrap();
|
||||
assert_eq!(abort[0].as_ref(), &[0x01]);
|
||||
let aborted_ids: Vec<String> = rmp_serde::from_slice(&abort[1]).unwrap();
|
||||
assert_eq!(aborted_ids, vec![request.request_id]);
|
||||
})
|
||||
},
|
||||
);
|
||||
|
||||
let llm = connect_async_llm_with_ipc(handshake_address, 0, "test-model", &ipc).await;
|
||||
let mut stream = llm.generate(sample_generate_request("req-drop", 4)).await.unwrap();
|
||||
|
||||
let output = stream.next().await.unwrap().unwrap();
|
||||
assert_eq!(output.token_ids, vec![99]);
|
||||
drop(stream);
|
||||
|
||||
let _ = shutdown_tx.send(());
|
||||
engine_task.await.unwrap();
|
||||
llm.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn duplicate_external_request_ids_are_randomized_before_reaching_engine_core_client() {
|
||||
let ipc = IpcNamespace::new().unwrap();
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
let engine_id = b"engine-dup".to_vec();
|
||||
|
||||
let (shutdown_tx, engine_task) = spawn_mock_engine_task(
|
||||
handshake_address.clone(),
|
||||
engine_id.clone(),
|
||||
|dealer, push| {
|
||||
Box::pin(async move {
|
||||
let add_1 = recv_engine_message(dealer).await;
|
||||
assert_eq!(add_1[0].as_ref(), &[0x00]);
|
||||
let request_1: EngineCoreRequest = rmp_serde::from_slice(&add_1[1]).unwrap();
|
||||
assert_eq!(request_1.external_req_id.as_deref(), Some("req-dup"));
|
||||
assert!(request_1.request_id.starts_with("req-dup-"));
|
||||
|
||||
let add_2 = recv_engine_message(dealer).await;
|
||||
assert_eq!(add_2[0].as_ref(), &[0x00]);
|
||||
let request_2: EngineCoreRequest = rmp_serde::from_slice(&add_2[1]).unwrap();
|
||||
assert_eq!(request_2.external_req_id.as_deref(), Some("req-dup"));
|
||||
assert!(request_2.request_id.starts_with("req-dup-"));
|
||||
assert_ne!(request_1.request_id, request_2.request_id);
|
||||
|
||||
send_outputs(
|
||||
push,
|
||||
EngineCoreOutputs {
|
||||
outputs: vec![request_output(
|
||||
&request_1.request_id,
|
||||
vec![],
|
||||
Some(EngineCoreFinishReason::Length),
|
||||
)],
|
||||
finished_requests: Some(BTreeSet::from([request_1.request_id.clone()])),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
send_outputs(
|
||||
push,
|
||||
EngineCoreOutputs {
|
||||
outputs: vec![request_output(
|
||||
&request_2.request_id,
|
||||
vec![],
|
||||
Some(EngineCoreFinishReason::Length),
|
||||
)],
|
||||
finished_requests: Some(BTreeSet::from([request_2.request_id])),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
})
|
||||
},
|
||||
);
|
||||
|
||||
let llm = connect_async_llm_with_ipc(handshake_address, 0, "test-model", &ipc).await;
|
||||
let stream_1 = llm.generate(sample_generate_request("req-dup", 1)).await.unwrap();
|
||||
let stream_2 = llm.generate(sample_generate_request("req-dup", 1)).await.unwrap();
|
||||
let internal_id_1 = stream_1.request_id().to_string();
|
||||
let internal_id_2 = stream_2.request_id().to_string();
|
||||
let collected_1 = stream_1.collect_output().await.unwrap();
|
||||
let collected_2 = stream_2.collect_output().await.unwrap();
|
||||
assert_eq!(collected_1.request_id, internal_id_1);
|
||||
assert_eq!(collected_2.request_id, internal_id_2);
|
||||
let _ = shutdown_tx.send(());
|
||||
engine_task.await.unwrap();
|
||||
llm.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn generate_records_request_metrics_in_prometheus_output() {
|
||||
let ipc = IpcNamespace::new().unwrap();
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
let engine_id = b"engine-metrics".to_vec();
|
||||
let model_name = request_metrics_model_name("metrics-model");
|
||||
|
||||
let (shutdown_tx, engine_task) = spawn_mock_engine_task(
|
||||
handshake_address.clone(),
|
||||
engine_id.clone(),
|
||||
|dealer, push| {
|
||||
Box::pin(async move {
|
||||
let add = recv_engine_message(dealer).await;
|
||||
assert_eq!(add[0].as_ref(), &[0x00]);
|
||||
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
|
||||
|
||||
send_outputs(
|
||||
push,
|
||||
EngineCoreOutputs {
|
||||
engine_index: 4,
|
||||
timestamp: 10.0,
|
||||
outputs: vec![EngineCoreOutput {
|
||||
prefill_stats: Some(PrefillStats {
|
||||
num_prompt_tokens: 2,
|
||||
num_computed_tokens: 2,
|
||||
..Default::default()
|
||||
}),
|
||||
..request_output_with_events(
|
||||
&request.request_id,
|
||||
vec![1],
|
||||
None,
|
||||
Some(vec![
|
||||
EngineCoreEvent {
|
||||
r#type: EngineCoreEventType::Queued,
|
||||
timestamp: 8.0,
|
||||
},
|
||||
EngineCoreEvent {
|
||||
r#type: EngineCoreEventType::Scheduled,
|
||||
timestamp: 9.0,
|
||||
},
|
||||
]),
|
||||
)
|
||||
}],
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
send_outputs(
|
||||
push,
|
||||
EngineCoreOutputs {
|
||||
engine_index: 4,
|
||||
timestamp: 11.5,
|
||||
outputs: vec![request_output_with_events(
|
||||
&request.request_id,
|
||||
vec![2, 3],
|
||||
Some(EngineCoreFinishReason::Length),
|
||||
Some(vec![EngineCoreEvent {
|
||||
r#type: EngineCoreEventType::Preempted,
|
||||
timestamp: 10.5,
|
||||
}]),
|
||||
)],
|
||||
finished_requests: Some(BTreeSet::from([request.request_id])),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
})
|
||||
},
|
||||
);
|
||||
|
||||
let llm = connect_async_llm_with_ipc(handshake_address, 0, &model_name, &ipc).await;
|
||||
let mut request = sample_generate_request("req-metrics", 8);
|
||||
request.arrival_time = None;
|
||||
let mut stream = llm.generate(request).await.unwrap();
|
||||
|
||||
assert_eq!(stream.next().await.unwrap().unwrap().token_ids, vec![1]);
|
||||
let final_output = stream.next().await.unwrap().unwrap();
|
||||
assert_eq!(final_output.token_ids, vec![2, 3]);
|
||||
assert_eq!(final_output.finish_reason, Some(FinishReason::Length));
|
||||
assert!(stream.next().await.is_none());
|
||||
|
||||
let rendered = METRICS.render().unwrap();
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:request_success_total{{model_name=\"{model_name}\",engine=\"4\",finished_reason=\"length\"}} 1"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:prompt_tokens_total{{model_name=\"{model_name}\",engine=\"4\"}} 2"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:prompt_tokens_by_source_total{{model_name=\"{model_name}\",engine=\"4\",source=\"local_compute\"}} 2"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:prompt_tokens_by_source_total{{model_name=\"{model_name}\",engine=\"4\",source=\"local_cache_hit\"}} 0"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:prompt_tokens_by_source_total{{model_name=\"{model_name}\",engine=\"4\",source=\"external_kv_transfer\"}} 0"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:prompt_tokens_cached_total{{model_name=\"{model_name}\",engine=\"4\"}} 0"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:generation_tokens_total{{model_name=\"{model_name}\",engine=\"4\"}} 3"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:num_preemptions_total{{model_name=\"{model_name}\",engine=\"4\"}} 1"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:time_to_first_token_seconds_count{{model_name=\"{model_name}\",engine=\"4\"}} 1"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:inter_token_latency_seconds_count{{model_name=\"{model_name}\",engine=\"4\"}} 1"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:e2e_request_latency_seconds_count{{model_name=\"{model_name}\",engine=\"4\"}} 1"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:request_prompt_tokens_count{{model_name=\"{model_name}\",engine=\"4\"}} 1"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:request_generation_tokens_count{{model_name=\"{model_name}\",engine=\"4\"}} 1"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:request_prefill_kv_computed_tokens_count{{model_name=\"{model_name}\",engine=\"4\"}} 1"
|
||||
)));
|
||||
|
||||
let _ = shutdown_tx.send(());
|
||||
engine_task.await.unwrap();
|
||||
llm.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn dropping_stream_records_abort_terminal_request_metrics() {
|
||||
let ipc = IpcNamespace::new().unwrap();
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
let engine_id = b"engine-metrics-drop".to_vec();
|
||||
let model_name = request_metrics_model_name("metrics-drop-model");
|
||||
|
||||
let (shutdown_tx, engine_task) = spawn_mock_engine_task(
|
||||
handshake_address.clone(),
|
||||
engine_id.clone(),
|
||||
|dealer, push| {
|
||||
Box::pin(async move {
|
||||
let add = recv_engine_message(dealer).await;
|
||||
assert_eq!(add[0].as_ref(), &[0x00]);
|
||||
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
|
||||
assert_eq!(request.external_req_id.as_deref(), Some("req-metrics-drop"));
|
||||
assert!(request.request_id.starts_with("req-metrics-drop-"));
|
||||
|
||||
send_outputs(
|
||||
push,
|
||||
EngineCoreOutputs {
|
||||
engine_index: 5,
|
||||
timestamp: 10.0,
|
||||
outputs: vec![request_output_with_events(
|
||||
&request.request_id,
|
||||
vec![99],
|
||||
None,
|
||||
Some(vec![
|
||||
EngineCoreEvent {
|
||||
r#type: EngineCoreEventType::Queued,
|
||||
timestamp: 8.0,
|
||||
},
|
||||
EngineCoreEvent {
|
||||
r#type: EngineCoreEventType::Scheduled,
|
||||
timestamp: 9.0,
|
||||
},
|
||||
]),
|
||||
)],
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
let abort =
|
||||
timeout(Duration::from_secs(1), recv_engine_message(dealer)).await.unwrap();
|
||||
assert_eq!(abort[0].as_ref(), &[0x01]);
|
||||
let aborted_ids: Vec<String> = rmp_serde::from_slice(&abort[1]).unwrap();
|
||||
assert_eq!(aborted_ids, vec![request.request_id]);
|
||||
})
|
||||
},
|
||||
);
|
||||
|
||||
let llm = connect_async_llm_with_ipc(handshake_address, 0, &model_name, &ipc).await;
|
||||
let mut request = sample_generate_request("req-metrics-drop", 8);
|
||||
request.arrival_time = None;
|
||||
let mut stream = llm.generate(request).await.unwrap();
|
||||
assert_eq!(stream.next().await.unwrap().unwrap().token_ids, vec![99]);
|
||||
drop(stream);
|
||||
|
||||
let _ = shutdown_tx.send(());
|
||||
engine_task.await.unwrap();
|
||||
let rendered = METRICS.render().unwrap();
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:request_success_total{{model_name=\"{model_name}\",engine=\"5\",finished_reason=\"abort\"}} 1"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:e2e_request_latency_seconds_count{{model_name=\"{model_name}\",engine=\"5\"}} 1"
|
||||
)));
|
||||
|
||||
llm.shutdown().await.unwrap();
|
||||
}
|
||||
Reference in New Issue
Block a user