[Rust Frontend] Extract shared options in route helper params (#44884)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-06-09 17:02:35 -07:00
committed by GitHub
parent d7607ad273
commit dac9e9a640
9 changed files with 224 additions and 279 deletions
+2 -1
View File
@@ -14,7 +14,8 @@ mod utils;
use std::sync::{Arc, OnceLock};
use anyhow::{Context as _, Result};
use axum::{Router, serve::ListenerExt as _};
use axum::Router;
use axum::serve::ListenerExt as _;
pub use config::{Config, CoordinatorMode, HttpListenerMode};
use tokio::net::TcpListener;
use tokio::time::{Instant, sleep_until};
@@ -22,7 +22,7 @@ use vllm_llm::{
CollectedGenerateOutput, FinishReason, GenerateOutput, GenerateOutputStreamExt as _,
};
use self::convert::prepare_generate_request;
use self::convert::{ResponseOptions, prepare_generate_request};
use self::types::{
GenerateLogprob, GenerateRequest, GenerateResponse, GenerateResponseChoice,
GenerateResponseStreamChoice, GenerateStreamResponse,
@@ -54,10 +54,7 @@ pub async fn generate(
);
let log_request = state.enable_log_requests;
let include_logprobs = prepared.include_logprobs;
let include_prompt_logprobs = prepared.include_prompt_logprobs;
let stream = prepared.stream;
let raw_stream = match state
.chat
.text()
@@ -80,9 +77,7 @@ pub async fn generate(
raw_stream,
prepared.request_id,
log_request,
prepared.include_usage,
prepared.include_continuous_usage,
include_logprobs,
prepared.options,
);
let sse_stream = generate_sse_stream(chunk_stream).instrument(request_span);
@@ -100,21 +95,11 @@ pub async fn generate(
}
};
if log_request {
info!(
parent: &request_span,
prompt_tokens = collected.prompt_token_ids.len(),
output_tokens = collected.token_ids.len(),
finish_reason = collected.finish_reason.as_str(),
"generate finished"
);
}
let response = match collect_generate(
collected,
prepared.request_id,
include_logprobs,
include_prompt_logprobs,
log_request,
prepared.options,
) {
Ok(response) => response,
Err(error) => return error.into_response(),
@@ -128,9 +113,13 @@ async fn generate_chunk_stream(
stream: impl Stream<Item = vllm_llm::Result<GenerateOutput>>,
request_id: String,
log_request: bool,
include_usage: bool,
include_continuous_usage: bool,
include_logprobs: bool,
ResponseOptions {
include_usage,
include_continuous_usage,
include_logprobs,
// Ignored: raw generate streaming has no prompt-logprobs wire shape.
include_prompt_logprobs: _,
}: ResponseOptions,
mut y: TryYielder<GenerateStreamResponse, ApiError>,
) -> Result<(), ApiError> {
pin_mut!(stream);
@@ -222,8 +211,15 @@ async fn generate_chunk_stream(
fn collect_generate(
collected: CollectedGenerateOutput,
request_id: String,
include_logprobs: bool,
include_prompt_logprobs: bool,
log_request: bool,
ResponseOptions {
// Ignored: non-streaming raw generate responses do not include usage.
include_usage: _,
// Ignored: continuous usage is a streaming-only option.
include_continuous_usage: _,
include_logprobs,
include_prompt_logprobs,
}: ResponseOptions,
) -> Result<GenerateResponse, ApiError> {
let logprobs = if include_logprobs {
let logprobs = collected.logprobs.as_ref().ok_or_else(|| {
@@ -246,13 +242,23 @@ fn collect_generate(
} else {
None
};
let finish_reason = collected.finish_reason.as_str().to_string();
if log_request {
info!(
prompt_tokens = collected.prompt_token_ids.len(),
output_tokens = collected.token_ids.len(),
%finish_reason,
"generate finished"
);
}
Ok(GenerateResponse {
request_id,
choices: vec![GenerateResponseChoice {
index: 0,
logprobs,
finish_reason: Some(collected.finish_reason.as_str().to_string()),
finish_reason: Some(finish_reason),
token_ids: collected.token_ids,
}],
prompt_logprobs,
@@ -408,11 +414,19 @@ mod tests {
}),
]);
let chunks: Vec<_> =
generate_chunk_stream(stream, "raw-stream".to_string(), false, true, true, false)
.try_collect()
.await
.expect("collect chunks");
let chunks: Vec<_> = generate_chunk_stream(
stream,
"raw-stream".to_string(),
false,
ResponseOptions {
include_usage: true,
include_continuous_usage: true,
..Default::default()
},
)
.try_collect()
.await
.expect("collect chunks");
assert_eq!(chunks.len(), 2);
assert_eq!(
@@ -8,19 +8,29 @@ use crate::utils::{ResolvedRequestContext, merge_kv_transfer_params};
/// Lowered generate request plus the response request ID.
#[derive(Debug, Clone, PartialEq)]
pub struct PreparedRequest {
pub(super) struct PreparedRequest {
pub request_id: String,
pub text_request: TextRequest,
pub stream: bool,
/// Public response rendering options for route-layer helpers.
pub options: ResponseOptions,
}
#[derive(Debug, Clone, Copy, Default, PartialEq)]
pub(super) struct ResponseOptions {
/// Whether the caller asked for the final streamed usage chunk.
pub include_usage: bool,
/// Whether the caller asked for usage on every streamed chunk.
pub include_continuous_usage: bool,
/// Whether the caller requested output logprobs on generate choices.
pub include_logprobs: bool,
/// Whether the caller requested top-level prompt logprobs.
pub include_prompt_logprobs: bool,
}
/// Validate and lower one raw generate request into the internal
/// text-generation format.
pub fn prepare_generate_request(
pub(super) fn prepare_generate_request(
request: GenerateRequest,
lora_resolution: &LoraModelResolution,
ctx: ResolvedRequestContext,
@@ -65,10 +75,12 @@ pub fn prepare_generate_request(
request_id: ctx.request_id,
text_request,
stream,
include_usage,
include_continuous_usage,
include_logprobs,
include_prompt_logprobs,
options: ResponseOptions {
include_usage,
include_continuous_usage,
include_logprobs,
include_prompt_logprobs,
},
})
}
@@ -158,7 +170,7 @@ mod tests {
)
.expect("prepare");
assert!(!prepared.include_usage);
assert!(!prepared.include_continuous_usage);
assert!(!prepared.options.include_usage);
assert!(!prepared.options.include_continuous_usage);
}
}
@@ -1,4 +1,4 @@
pub mod convert;
mod convert;
mod types;
mod validate;
@@ -23,8 +23,8 @@ use vllm_chat::{
};
use vllm_engine_core_client::protocol::StopReason;
use self::convert::{ResponseOptions, prepare_chat_request};
use crate::error::{ApiError, bail_server_error, server_error};
use crate::routes::openai::chat_completions::convert::prepare_chat_request;
use crate::routes::openai::chat_completions::types::{
AssistantRole, ChatCompletionChoice, ChatCompletionMessage, ChatCompletionRequest,
ChatCompletionResponse, ChatCompletionStreamChoice, ChatCompletionStreamResponse,
@@ -83,12 +83,7 @@ pub async fn chat_completions(
prepared.response_model,
created,
log_request,
prepared.include_usage,
prepared.requested_logprobs,
prepared.include_reasoning,
prepared.echo,
prepared.return_token_ids,
prepared.return_tokens_as_token_ids,
prepared.options,
);
let sse_stream = chat_completion_sse_stream(chunk_stream).instrument(request_span);
@@ -99,12 +94,8 @@ pub async fn chat_completions(
prepared.request_id,
prepared.response_model,
created,
prepared.requested_logprobs,
prepared.include_prompt_logprobs,
prepared.include_reasoning,
prepared.echo,
prepared.return_token_ids,
prepared.return_tokens_as_token_ids,
log_request,
prepared.options,
)
.instrument(request_span.clone())
.await
@@ -113,18 +104,6 @@ pub async fn chat_completions(
Err(error) => return error.into_response(),
};
if log_request {
let usage = response.usage.as_ref();
info!(
parent: &request_span,
model = %response.model,
prompt_tokens = usage.map_or(0, |u| u.prompt_tokens),
output_tokens = usage.and_then(|u| u.completion_tokens).unwrap_or(0),
finish_reason = response.choices.first().and_then(|c| c.finish_reason.as_deref()).unwrap_or("unknown"),
"chat completion finished"
);
}
Json(response).into_response()
}
}
@@ -134,12 +113,17 @@ async fn collect_chat_completion(
request_id: String,
response_model: String,
created: u64,
requested_logprobs: bool,
include_prompt_logprobs: bool,
include_reasoning: bool,
echo: Option<String>,
return_token_ids: bool,
return_tokens_as_token_ids: bool,
log_request: bool,
ResponseOptions {
// Ignored: non-streaming responses always include usage.
include_usage: _,
requested_logprobs,
include_prompt_logprobs,
include_reasoning,
echo,
return_token_ids,
return_tokens_as_token_ids,
}: ResponseOptions,
) -> Result<ChatCompletionResponse, ApiError> {
let collected = stream.collect_message().await.map_err(|error| {
server_error!(
@@ -201,6 +185,16 @@ async fn collect_chat_completion(
};
let usage = Usage::from_counts(prompt_token_count as u32, output_token_count as u32);
if log_request {
info!(
model = %response_model,
prompt_tokens = usage.prompt_tokens,
output_tokens = usage.completion_tokens.unwrap_or(0),
finish_reason = %finish_reason,
"chat completion finished"
);
}
Ok(ChatCompletionResponse {
id: request_id,
object: "chat.completion".to_string(),
@@ -238,12 +232,16 @@ async fn chat_completion_chunk_stream(
response_model: String,
created: u64,
log_request: bool,
include_usage: bool,
requested_logprobs: bool,
include_reasoning: bool,
echo: Option<String>,
return_token_ids: bool,
return_tokens_as_token_ids: bool,
ResponseOptions {
include_usage,
requested_logprobs,
// Ignored: chat streaming prompt logprobs are rejected for Python parity.
include_prompt_logprobs: _,
include_reasoning,
echo,
return_token_ids,
return_tokens_as_token_ids,
}: ResponseOptions,
mut y: TryYielder<ChatCompletionStreamResponse, ApiError>,
) -> Result<(), ApiError> {
let mut saw_tool_calls = false;
@@ -806,7 +804,7 @@ mod tests {
use vllm_engine_core_client::protocol::StopReason;
use vllm_text::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTokenLogprob};
use super::{block_delta_chunk, chat_completion_chunk_stream, final_chunk};
use super::{ResponseOptions, block_delta_chunk, chat_completion_chunk_stream, final_chunk};
#[test]
fn text_chunk_uses_content_only_delta() {
@@ -932,12 +930,11 @@ mod tests {
"model".to_string(),
1,
false,
false,
true,
true,
None,
false,
false,
ResponseOptions {
requested_logprobs: true,
include_reasoning: true,
..Default::default()
},
)
.collect::<Vec<_>>()
.await
@@ -996,12 +993,11 @@ mod tests {
"model".to_string(),
1,
false,
false,
true,
true,
None,
false,
false,
ResponseOptions {
requested_logprobs: true,
include_reasoning: true,
..Default::default()
},
)
.collect::<Vec<_>>()
.await
@@ -1049,12 +1045,7 @@ mod tests {
"model".to_string(),
1,
false,
false,
false,
false,
None,
false,
false,
ResponseOptions::default(),
)
.collect::<Vec<_>>()
.await
@@ -1132,12 +1123,11 @@ mod tests {
"model".to_string(),
1,
false,
false,
true,
false,
None,
true,
false,
ResponseOptions {
requested_logprobs: true,
return_token_ids: true,
..Default::default()
},
)
.collect::<Vec<_>>()
.await
@@ -1263,12 +1253,11 @@ mod tests {
"model".to_string(),
1,
false,
false,
true,
false,
None,
true,
false,
ResponseOptions {
requested_logprobs: true,
return_token_ids: true,
..Default::default()
},
)
.collect::<Vec<_>>()
.await
@@ -1342,12 +1331,10 @@ mod tests {
"model".to_string(),
1,
false,
false,
false,
true,
None,
false,
false,
ResponseOptions {
include_reasoning: true,
..Default::default()
},
)
.collect::<Vec<_>>()
.await
@@ -18,11 +18,19 @@ use crate::utils::{ResolvedRequestContext, convert_logit_bias, merge_kv_transfer
/// Lowered chat request plus the public response metadata carried by every SSE
/// chunk.
#[derive(Debug, Clone, PartialEq)]
pub struct PreparedRequest {
pub(super) struct PreparedRequest {
/// Stable OpenAI-style request ID, reused as the external chat request ID.
pub request_id: String,
/// Public model ID echoed back to the client.
pub response_model: String,
/// Public response rendering options for route-layer helpers.
pub options: ResponseOptions,
/// Lowered chat request for `vllm-chat`.
pub chat_request: ChatRequest,
}
#[derive(Debug, Clone, Default, PartialEq)]
pub(super) struct ResponseOptions {
/// Whether the caller asked for the final streamed usage chunk.
pub include_usage: bool,
/// Whether the caller requested output logprobs on chat choices.
@@ -31,8 +39,6 @@ pub struct PreparedRequest {
pub include_prompt_logprobs: bool,
/// Whether to include reasoning content in OpenAI responses.
pub include_reasoning: bool,
/// Lowered chat request for `vllm-chat`.
pub chat_request: ChatRequest,
/// Last assistant-role message content to echo back when `echo=true`.
pub echo: Option<String>,
/// Whether to include token IDs alongside generated text.
@@ -46,7 +52,7 @@ pub struct PreparedRequest {
///
/// `lora_resolution.model_names` must be non-empty; the first entry is used as
/// the base `model` field in responses when no LoRA adapter is selected.
pub(crate) fn prepare_chat_request(
pub(super) fn prepare_chat_request(
request: ChatCompletionRequest,
lora_resolution: &LoraModelResolution,
ctx: ResolvedRequestContext,
@@ -146,14 +152,16 @@ pub(crate) fn prepare_chat_request(
Ok(PreparedRequest {
request_id,
response_model,
include_usage,
requested_logprobs,
include_prompt_logprobs,
include_reasoning,
options: ResponseOptions {
include_usage,
requested_logprobs,
include_prompt_logprobs,
include_reasoning,
echo,
return_token_ids: request.return_token_ids.unwrap_or(false),
return_tokens_as_token_ids: request.return_tokens_as_token_ids.unwrap_or(false),
},
chat_request,
echo,
return_token_ids: request.return_token_ids.unwrap_or(false),
return_tokens_as_token_ids: request.return_tokens_as_token_ids.unwrap_or(false),
})
}
pub(crate) fn normalize_generation_prompt_mode(
@@ -497,7 +505,7 @@ mod tests {
)
.expect("request is valid");
assert!(!prepared.include_reasoning);
assert!(!prepared.options.include_reasoning);
}
#[test]
@@ -867,8 +875,8 @@ mod tests {
)
.expect("request is valid");
assert!(prepared.requested_logprobs);
assert!(prepared.include_prompt_logprobs);
assert!(prepared.options.requested_logprobs);
assert!(prepared.options.include_prompt_logprobs);
assert_eq!(prepared.chat_request.sampling_params.logprobs, Some(0));
assert_eq!(
prepared.chat_request.sampling_params.prompt_logprobs,
@@ -894,7 +902,7 @@ mod tests {
assert_eq!(prepared.chat_request.sampling_params.logprobs, Some(3));
assert_eq!(prepared.chat_request.sampling_params.prompt_logprobs, None);
assert!(!prepared.include_prompt_logprobs);
assert!(!prepared.options.include_prompt_logprobs);
}
#[test]
@@ -18,13 +18,13 @@ use tracing::{debug, error, info, trace};
use tracing_futures::Instrument as _;
use vllm_text::{DecodedTextEvent, FinishReason, TextOutputStream, TextOutputStreamExt as _};
use self::convert::{ResponseOptions, prepare_completion_request};
use super::utils::logprobs::{
collected_logprobs_to_openai, decoded_logprobs_to_openai, decoded_prompt_logprobs_to_maps,
text_len,
};
use super::utils::types::Usage;
use crate::error::{ApiError, bail_server_error, server_error};
use crate::routes::openai::completions::convert::prepare_completion_request;
use crate::routes::openai::completions::types::{
CompletionChoice, CompletionRequest, CompletionResponse, CompletionSseChunk,
CompletionStreamChoice, CompletionStreamResponse,
@@ -42,7 +42,6 @@ pub async fn completions(
ValidatedJson(body): ValidatedJson<CompletionRequest>,
) -> Response {
let stream = body.stream;
let logprobs = body.logprobs;
let request_context = resolve_request_context(&headers, body.request_id.as_deref());
let lora_resolution = state.resolve_model_with_loras(Some(&body.model)).await;
@@ -57,9 +56,7 @@ pub async fn completions(
);
let created = unix_timestamp();
let include_prompt_logprobs = prepared.text_request.sampling_params.prompt_logprobs.is_some();
let log_request = state.enable_log_requests;
let text_stream = match state
.chat
.text()
@@ -84,11 +81,7 @@ pub async fn completions(
prepared.response_model,
created,
log_request,
prepared.include_usage,
prepared.echo,
logprobs,
prepared.return_token_ids,
prepared.return_tokens_as_token_ids,
prepared.options,
);
let sse_stream = completion_sse_stream(chunk_stream).instrument(request_span);
@@ -99,11 +92,8 @@ pub async fn completions(
prepared.request_id,
prepared.response_model,
created,
prepared.echo,
logprobs,
include_prompt_logprobs,
prepared.return_token_ids,
prepared.return_tokens_as_token_ids,
log_request,
prepared.options,
)
.instrument(request_span.clone())
.await
@@ -112,18 +102,6 @@ pub async fn completions(
Err(error) => return error.into_response(),
};
if log_request {
let usage = response.usage.as_ref();
info!(
parent: &request_span,
model = %response.model,
prompt_tokens = usage.map_or(0, |u| u.prompt_tokens),
output_tokens = usage.and_then(|u| u.completion_tokens).unwrap_or(0),
finish_reason = response.choices.first().and_then(|c| c.finish_reason.as_deref()).unwrap_or("unknown"),
"completion finished"
);
}
Json(response).into_response()
}
}
@@ -133,11 +111,16 @@ async fn collect_completion(
request_id: String,
response_model: String,
created: u64,
echo: Option<String>,
requested_logprobs: Option<u32>,
include_prompt_logprobs: bool,
return_token_ids: bool,
return_tokens_as_token_ids: bool,
log_request: bool,
ResponseOptions {
// Ignored: non-streaming responses always include usage.
include_usage: _,
echo,
requested_logprobs,
include_prompt_logprobs,
return_token_ids,
return_tokens_as_token_ids,
}: ResponseOptions,
) -> Result<CompletionResponse, ApiError> {
let collected = stream
.collect_output()
@@ -175,6 +158,21 @@ async fn collect_completion(
None => collected.text,
Some(prompt) => format!("{prompt}{}", collected.text),
};
let finish_reason = completion_finish_reason_to_openai(finish_reason)?.to_string();
let usage = Usage::from_counts(
collected.prompt_token_ids.len() as u32,
collected.token_ids.len() as u32,
);
if log_request {
info!(
model = %response_model,
prompt_tokens = usage.prompt_tokens,
output_tokens = usage.completion_tokens.unwrap_or(0),
%finish_reason,
"completion finished"
);
}
Ok(CompletionResponse {
id: request_id,
@@ -185,16 +183,13 @@ async fn collect_completion(
index: 0,
text,
logprobs,
finish_reason: Some(completion_finish_reason_to_openai(finish_reason)?.into()),
finish_reason: Some(finish_reason),
stop_reason,
prompt_logprobs,
token_ids: return_token_ids.then(|| collected.token_ids.clone()),
prompt_token_ids: return_token_ids.then(|| collected.prompt_token_ids.to_vec()),
}],
usage: Some(Usage::from_counts(
collected.prompt_token_ids.len() as u32,
collected.token_ids.len() as u32,
)),
usage: Some(usage),
system_fingerprint: None,
kv_transfer_params: collected.kv_transfer_params,
})
@@ -208,11 +203,15 @@ async fn completion_chunk_stream(
response_model: String,
created: u64,
log_request: bool,
include_usage: bool,
echo: Option<String>,
requested_logprobs: Option<u32>,
return_token_ids: bool,
return_tokens_as_token_ids: bool,
ResponseOptions {
include_usage,
echo,
requested_logprobs,
// Ignored: streaming prompt logprobs are rejected for Python parity.
include_prompt_logprobs: _,
return_token_ids,
return_tokens_as_token_ids,
}: ResponseOptions,
mut y: TryYielder<CompletionSseChunk, ApiError>,
) -> Result<(), ApiError> {
pin_mut!(stream);
@@ -432,7 +431,7 @@ mod tests {
FinishReason, Finished,
};
use super::{CompletionSseChunk, completion_chunk_stream, final_chunk};
use super::{CompletionSseChunk, ResponseOptions, completion_chunk_stream, final_chunk};
#[test]
fn final_chunk_maps_stop_finish_reason() {
@@ -527,11 +526,10 @@ mod tests {
"model".to_string(),
1,
false,
false,
None,
Some(1),
false,
false,
ResponseOptions {
requested_logprobs: Some(1),
..Default::default()
},
)
.collect::<Vec<_>>()
.await;
@@ -10,18 +10,28 @@ use crate::utils::{ResolvedRequestContext, convert_logit_bias, merge_kv_transfer
/// Lowered completion request plus the public response metadata carried by
/// every SSE chunk.
#[derive(Debug, Clone, PartialEq)]
pub struct PreparedRequest {
pub(super) struct PreparedRequest {
/// Stable OpenAI-style request ID, reused as the external text request ID.
pub request_id: String,
/// Public model ID echoed back to the client.
pub response_model: String,
/// Whether the caller asked for the final streamed usage chunk.
pub include_usage: bool,
/// Public response rendering options for route-layer helpers.
pub options: ResponseOptions,
/// Lowered text request for the shared `vllm-text` facade.
pub text_request: TextRequest,
}
#[derive(Debug, Clone, Default, PartialEq)]
pub(super) struct ResponseOptions {
/// Whether the caller asked for the final streamed usage chunk.
pub include_usage: bool,
/// Original text prompt that should be echoed back northbound when
/// `echo=true`.
pub echo: Option<String>,
/// Whether the caller requested output logprobs on completion choices.
pub requested_logprobs: Option<u32>,
/// Whether the caller requested choice-level prompt logprobs.
pub include_prompt_logprobs: bool,
/// Whether to include token IDs alongside generated text.
pub return_token_ids: bool,
/// Whether to format logprob tokens as `token_id:{id}`.
@@ -33,7 +43,7 @@ pub struct PreparedRequest {
///
/// `lora_resolution.model_names` must be non-empty; the first entry is used as
/// the base `model` field in responses when no LoRA adapter is selected.
pub(crate) fn prepare_completion_request(
pub(super) fn prepare_completion_request(
request: CompletionRequest,
lora_resolution: &LoraModelResolution,
ctx: ResolvedRequestContext,
@@ -64,6 +74,7 @@ pub(crate) fn prepare_completion_request(
let include_usage = (request.stream_options.as_ref())
.and_then(|options| options.include_usage)
.unwrap_or(false);
let include_prompt_logprobs = prompt_logprobs.is_some();
let echo = request.echo.then(|| request.prompt.as_text().cloned()).flatten();
let structured_outputs =
@@ -116,11 +127,15 @@ pub(crate) fn prepare_completion_request(
Ok(PreparedRequest {
request_id,
response_model,
include_usage,
options: ResponseOptions {
include_usage,
echo,
requested_logprobs: request.logprobs,
include_prompt_logprobs,
return_token_ids: request.return_token_ids.unwrap_or(false),
return_tokens_as_token_ids: request.return_tokens_as_token_ids.unwrap_or(false),
},
text_request,
echo,
return_token_ids: request.return_token_ids.unwrap_or(false),
return_tokens_as_token_ids: request.return_tokens_as_token_ids.unwrap_or(false),
})
}
@@ -206,7 +221,7 @@ mod tests {
)
.expect("prepare");
assert!(prepared.include_usage);
assert!(prepared.options.include_usage);
assert_eq!(
prepared.text_request.prompt,
Prompt::TokenIds(vec![11, 22, 33])
@@ -250,7 +265,7 @@ mod tests {
)
.expect("prepare");
assert_eq!(prepared.echo, Some("hello".to_string()));
assert_eq!(prepared.options.echo, Some("hello".to_string()));
assert_eq!(prepared.text_request.sampling_params.max_tokens, Some(7));
}
+3 -92
View File
@@ -14,15 +14,14 @@ use std::{fmt, fs};
use axum::body::{Body, to_bytes};
use axum::http::{Request, StatusCode};
use bytes::Bytes;
use futures::StreamExt as _;
use rmpv::Value;
use serde_json::json;
use serial_test::serial;
use tower::{Service as _, ServiceExt as _};
use vllm_chat::{
ChatBackend, ChatContent, ChatContentPart, ChatEvent, ChatLlm, ChatMessage, ChatRenderer,
ChatRequest, ChatRole, ChatTextBackend, DefaultChatOutputProcessor, DynChatOutputProcessor,
DynChatRenderer, NewChatOutputProcessorOptions, SamplingParams,
ChatBackend, ChatContent, ChatContentPart, ChatLlm, ChatMessage, ChatRenderer, ChatRequest,
ChatTextBackend, DefaultChatOutputProcessor, DynChatOutputProcessor, DynChatRenderer,
NewChatOutputProcessorOptions,
};
use vllm_engine_core_client::protocol::logprobs::{
Logprobs, MaybeWireLogprobs, PositionLogprobs, TokenLogprob,
@@ -44,8 +43,6 @@ use zeromq::prelude::{SocketRecv, SocketSend};
use zeromq::{DealerSocket, PushSocket, ZmqMessage};
use super::{build_router, build_router_with_dev_mode, build_router_with_dev_mode_and_lora};
use crate::lora::LoraModelResolution;
use crate::routes::openai::chat_completions::convert::prepare_chat_request;
use crate::state::AppState;
fn request_output(
@@ -3478,92 +3475,6 @@ async fn completions_echo_stream_emits_separate_prompt_chunk() {
assert_eq!(usage_chunk["usage"]["completion_tokens"], 3);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn chat_harness_streams_text_events() {
let (chat, engine_task) = test_chat_with_engine_handle().await;
let mut stream = chat
.chat(ChatRequest {
messages: vec![ChatMessage::text(ChatRole::User, "hello")],
sampling_params: SamplingParams {
max_tokens: Some(8),
..Default::default()
},
request_id: "chat-harness".to_string(),
..ChatRequest::for_test()
})
.await
.expect("submit chat request");
let mut saw_text = false;
let mut saw_done = false;
while let Some(event) = stream.next().await {
match event.expect("chat event") {
ChatEvent::BlockDelta { .. } => saw_text = true,
ChatEvent::Done { .. } => {
saw_done = true;
break;
}
ChatEvent::Start { .. }
| ChatEvent::LogprobsDelta { .. }
| ChatEvent::BlockStart { .. }
| ChatEvent::BlockEnd { .. }
| ChatEvent::ToolCallStart { .. }
| ChatEvent::ToolCallArgumentsDelta { .. }
| ChatEvent::ToolCallEnd { .. } => {}
}
}
engine_task.await.expect("mock engine task");
assert!(saw_text);
assert!(saw_done);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn prepared_openai_request_streams_text_events() {
let (chat, engine_task) = test_chat_with_engine_handle().await;
let prepared = prepare_chat_request(
serde_json::from_value(json!({
"model": "Qwen/Qwen1.5-0.5B-Chat",
"stream": true,
"messages": [{"role": "user", "content": "hello"}]
}))
.expect("decode request"),
&LoraModelResolution {
model_names: vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
lora_request: None,
},
crate::utils::ResolvedRequestContext::default(),
)
.expect("prepare request");
let mut stream = chat.chat(prepared.chat_request).await.expect("submit chat request");
let mut saw_text = false;
let mut saw_done = false;
while let Some(event) = stream.next().await {
match event.expect("chat event") {
ChatEvent::BlockDelta { .. } => saw_text = true,
ChatEvent::Done { .. } => {
saw_done = true;
break;
}
ChatEvent::Start { .. }
| ChatEvent::LogprobsDelta { .. }
| ChatEvent::BlockStart { .. }
| ChatEvent::BlockEnd { .. }
| ChatEvent::ToolCallStart { .. }
| ChatEvent::ToolCallArgumentsDelta { .. }
| ChatEvent::ToolCallEnd { .. } => {}
}
}
engine_task.await.expect("mock engine task");
assert!(saw_text);
assert!(saw_done);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn reasoning_blocks_are_mapped_to_reasoning_sse_chunks() {
-1
View File
@@ -10,7 +10,6 @@ use vllm_engine_core_client::EngineCoreClient;
use vllm_engine_core_client::protocol::lora::LoraRequest;
use crate::lora::{LoadLoraError, LoraManager, LoraModelResolution, UnloadLoraError};
use crate::server_info::{ServerInfoConfigFormat, ServerInfoSnapshot};
const SHUTDOWN_REFCOUNT_POLL_INTERVAL: Duration = Duration::from_millis(100);