diff --git a/rust/src/server/src/lib.rs b/rust/src/server/src/lib.rs index 94421548aa..e2c17cc262 100644 --- a/rust/src/server/src/lib.rs +++ b/rust/src/server/src/lib.rs @@ -14,7 +14,8 @@ mod utils; use std::sync::{Arc, OnceLock}; use anyhow::{Context as _, Result}; -use axum::{Router, serve::ListenerExt as _}; +use axum::Router; +use axum::serve::ListenerExt as _; pub use config::{Config, CoordinatorMode, HttpListenerMode}; use tokio::net::TcpListener; use tokio::time::{Instant, sleep_until}; diff --git a/rust/src/server/src/routes/inference/generate.rs b/rust/src/server/src/routes/inference/generate.rs index f15f757c09..5b675f39df 100644 --- a/rust/src/server/src/routes/inference/generate.rs +++ b/rust/src/server/src/routes/inference/generate.rs @@ -22,7 +22,7 @@ use vllm_llm::{ CollectedGenerateOutput, FinishReason, GenerateOutput, GenerateOutputStreamExt as _, }; -use self::convert::prepare_generate_request; +use self::convert::{ResponseOptions, prepare_generate_request}; use self::types::{ GenerateLogprob, GenerateRequest, GenerateResponse, GenerateResponseChoice, GenerateResponseStreamChoice, GenerateStreamResponse, @@ -54,10 +54,7 @@ pub async fn generate( ); let log_request = state.enable_log_requests; - let include_logprobs = prepared.include_logprobs; - let include_prompt_logprobs = prepared.include_prompt_logprobs; let stream = prepared.stream; - let raw_stream = match state .chat .text() @@ -80,9 +77,7 @@ pub async fn generate( raw_stream, prepared.request_id, log_request, - prepared.include_usage, - prepared.include_continuous_usage, - include_logprobs, + prepared.options, ); let sse_stream = generate_sse_stream(chunk_stream).instrument(request_span); @@ -100,21 +95,11 @@ pub async fn generate( } }; - if log_request { - info!( - parent: &request_span, - prompt_tokens = collected.prompt_token_ids.len(), - output_tokens = collected.token_ids.len(), - finish_reason = collected.finish_reason.as_str(), - "generate finished" - ); - } - let response = match collect_generate( collected, prepared.request_id, - include_logprobs, - include_prompt_logprobs, + log_request, + prepared.options, ) { Ok(response) => response, Err(error) => return error.into_response(), @@ -128,9 +113,13 @@ async fn generate_chunk_stream( stream: impl Stream>, request_id: String, log_request: bool, - include_usage: bool, - include_continuous_usage: bool, - include_logprobs: bool, + ResponseOptions { + include_usage, + include_continuous_usage, + include_logprobs, + // Ignored: raw generate streaming has no prompt-logprobs wire shape. + include_prompt_logprobs: _, + }: ResponseOptions, mut y: TryYielder, ) -> Result<(), ApiError> { pin_mut!(stream); @@ -222,8 +211,15 @@ async fn generate_chunk_stream( fn collect_generate( collected: CollectedGenerateOutput, request_id: String, - include_logprobs: bool, - include_prompt_logprobs: bool, + log_request: bool, + ResponseOptions { + // Ignored: non-streaming raw generate responses do not include usage. + include_usage: _, + // Ignored: continuous usage is a streaming-only option. + include_continuous_usage: _, + include_logprobs, + include_prompt_logprobs, + }: ResponseOptions, ) -> Result { let logprobs = if include_logprobs { let logprobs = collected.logprobs.as_ref().ok_or_else(|| { @@ -246,13 +242,23 @@ fn collect_generate( } else { None }; + let finish_reason = collected.finish_reason.as_str().to_string(); + + if log_request { + info!( + prompt_tokens = collected.prompt_token_ids.len(), + output_tokens = collected.token_ids.len(), + %finish_reason, + "generate finished" + ); + } Ok(GenerateResponse { request_id, choices: vec![GenerateResponseChoice { index: 0, logprobs, - finish_reason: Some(collected.finish_reason.as_str().to_string()), + finish_reason: Some(finish_reason), token_ids: collected.token_ids, }], prompt_logprobs, @@ -408,11 +414,19 @@ mod tests { }), ]); - let chunks: Vec<_> = - generate_chunk_stream(stream, "raw-stream".to_string(), false, true, true, false) - .try_collect() - .await - .expect("collect chunks"); + let chunks: Vec<_> = generate_chunk_stream( + stream, + "raw-stream".to_string(), + false, + ResponseOptions { + include_usage: true, + include_continuous_usage: true, + ..Default::default() + }, + ) + .try_collect() + .await + .expect("collect chunks"); assert_eq!(chunks.len(), 2); assert_eq!( diff --git a/rust/src/server/src/routes/inference/generate/convert.rs b/rust/src/server/src/routes/inference/generate/convert.rs index f87ff403a7..73bca4a1f8 100644 --- a/rust/src/server/src/routes/inference/generate/convert.rs +++ b/rust/src/server/src/routes/inference/generate/convert.rs @@ -8,19 +8,29 @@ use crate::utils::{ResolvedRequestContext, merge_kv_transfer_params}; /// Lowered generate request plus the response request ID. #[derive(Debug, Clone, PartialEq)] -pub struct PreparedRequest { +pub(super) struct PreparedRequest { pub request_id: String, pub text_request: TextRequest, pub stream: bool, + /// Public response rendering options for route-layer helpers. + pub options: ResponseOptions, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub(super) struct ResponseOptions { + /// Whether the caller asked for the final streamed usage chunk. pub include_usage: bool, + /// Whether the caller asked for usage on every streamed chunk. pub include_continuous_usage: bool, + /// Whether the caller requested output logprobs on generate choices. pub include_logprobs: bool, + /// Whether the caller requested top-level prompt logprobs. pub include_prompt_logprobs: bool, } /// Validate and lower one raw generate request into the internal /// text-generation format. -pub fn prepare_generate_request( +pub(super) fn prepare_generate_request( request: GenerateRequest, lora_resolution: &LoraModelResolution, ctx: ResolvedRequestContext, @@ -65,10 +75,12 @@ pub fn prepare_generate_request( request_id: ctx.request_id, text_request, stream, - include_usage, - include_continuous_usage, - include_logprobs, - include_prompt_logprobs, + options: ResponseOptions { + include_usage, + include_continuous_usage, + include_logprobs, + include_prompt_logprobs, + }, }) } @@ -158,7 +170,7 @@ mod tests { ) .expect("prepare"); - assert!(!prepared.include_usage); - assert!(!prepared.include_continuous_usage); + assert!(!prepared.options.include_usage); + assert!(!prepared.options.include_continuous_usage); } } diff --git a/rust/src/server/src/routes/openai/chat_completions.rs b/rust/src/server/src/routes/openai/chat_completions.rs index 0cccd8bf4a..e58f2e2ac7 100644 --- a/rust/src/server/src/routes/openai/chat_completions.rs +++ b/rust/src/server/src/routes/openai/chat_completions.rs @@ -1,4 +1,4 @@ -pub mod convert; +mod convert; mod types; mod validate; @@ -23,8 +23,8 @@ use vllm_chat::{ }; use vllm_engine_core_client::protocol::StopReason; +use self::convert::{ResponseOptions, prepare_chat_request}; use crate::error::{ApiError, bail_server_error, server_error}; -use crate::routes::openai::chat_completions::convert::prepare_chat_request; use crate::routes::openai::chat_completions::types::{ AssistantRole, ChatCompletionChoice, ChatCompletionMessage, ChatCompletionRequest, ChatCompletionResponse, ChatCompletionStreamChoice, ChatCompletionStreamResponse, @@ -83,12 +83,7 @@ pub async fn chat_completions( prepared.response_model, created, log_request, - prepared.include_usage, - prepared.requested_logprobs, - prepared.include_reasoning, - prepared.echo, - prepared.return_token_ids, - prepared.return_tokens_as_token_ids, + prepared.options, ); let sse_stream = chat_completion_sse_stream(chunk_stream).instrument(request_span); @@ -99,12 +94,8 @@ pub async fn chat_completions( prepared.request_id, prepared.response_model, created, - prepared.requested_logprobs, - prepared.include_prompt_logprobs, - prepared.include_reasoning, - prepared.echo, - prepared.return_token_ids, - prepared.return_tokens_as_token_ids, + log_request, + prepared.options, ) .instrument(request_span.clone()) .await @@ -113,18 +104,6 @@ pub async fn chat_completions( Err(error) => return error.into_response(), }; - if log_request { - let usage = response.usage.as_ref(); - info!( - parent: &request_span, - model = %response.model, - prompt_tokens = usage.map_or(0, |u| u.prompt_tokens), - output_tokens = usage.and_then(|u| u.completion_tokens).unwrap_or(0), - finish_reason = response.choices.first().and_then(|c| c.finish_reason.as_deref()).unwrap_or("unknown"), - "chat completion finished" - ); - } - Json(response).into_response() } } @@ -134,12 +113,17 @@ async fn collect_chat_completion( request_id: String, response_model: String, created: u64, - requested_logprobs: bool, - include_prompt_logprobs: bool, - include_reasoning: bool, - echo: Option, - return_token_ids: bool, - return_tokens_as_token_ids: bool, + log_request: bool, + ResponseOptions { + // Ignored: non-streaming responses always include usage. + include_usage: _, + requested_logprobs, + include_prompt_logprobs, + include_reasoning, + echo, + return_token_ids, + return_tokens_as_token_ids, + }: ResponseOptions, ) -> Result { let collected = stream.collect_message().await.map_err(|error| { server_error!( @@ -201,6 +185,16 @@ async fn collect_chat_completion( }; let usage = Usage::from_counts(prompt_token_count as u32, output_token_count as u32); + if log_request { + info!( + model = %response_model, + prompt_tokens = usage.prompt_tokens, + output_tokens = usage.completion_tokens.unwrap_or(0), + finish_reason = %finish_reason, + "chat completion finished" + ); + } + Ok(ChatCompletionResponse { id: request_id, object: "chat.completion".to_string(), @@ -238,12 +232,16 @@ async fn chat_completion_chunk_stream( response_model: String, created: u64, log_request: bool, - include_usage: bool, - requested_logprobs: bool, - include_reasoning: bool, - echo: Option, - return_token_ids: bool, - return_tokens_as_token_ids: bool, + ResponseOptions { + include_usage, + requested_logprobs, + // Ignored: chat streaming prompt logprobs are rejected for Python parity. + include_prompt_logprobs: _, + include_reasoning, + echo, + return_token_ids, + return_tokens_as_token_ids, + }: ResponseOptions, mut y: TryYielder, ) -> Result<(), ApiError> { let mut saw_tool_calls = false; @@ -806,7 +804,7 @@ mod tests { use vllm_engine_core_client::protocol::StopReason; use vllm_text::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTokenLogprob}; - use super::{block_delta_chunk, chat_completion_chunk_stream, final_chunk}; + use super::{ResponseOptions, block_delta_chunk, chat_completion_chunk_stream, final_chunk}; #[test] fn text_chunk_uses_content_only_delta() { @@ -932,12 +930,11 @@ mod tests { "model".to_string(), 1, false, - false, - true, - true, - None, - false, - false, + ResponseOptions { + requested_logprobs: true, + include_reasoning: true, + ..Default::default() + }, ) .collect::>() .await @@ -996,12 +993,11 @@ mod tests { "model".to_string(), 1, false, - false, - true, - true, - None, - false, - false, + ResponseOptions { + requested_logprobs: true, + include_reasoning: true, + ..Default::default() + }, ) .collect::>() .await @@ -1049,12 +1045,7 @@ mod tests { "model".to_string(), 1, false, - false, - false, - false, - None, - false, - false, + ResponseOptions::default(), ) .collect::>() .await @@ -1132,12 +1123,11 @@ mod tests { "model".to_string(), 1, false, - false, - true, - false, - None, - true, - false, + ResponseOptions { + requested_logprobs: true, + return_token_ids: true, + ..Default::default() + }, ) .collect::>() .await @@ -1263,12 +1253,11 @@ mod tests { "model".to_string(), 1, false, - false, - true, - false, - None, - true, - false, + ResponseOptions { + requested_logprobs: true, + return_token_ids: true, + ..Default::default() + }, ) .collect::>() .await @@ -1342,12 +1331,10 @@ mod tests { "model".to_string(), 1, false, - false, - false, - true, - None, - false, - false, + ResponseOptions { + include_reasoning: true, + ..Default::default() + }, ) .collect::>() .await diff --git a/rust/src/server/src/routes/openai/chat_completions/convert.rs b/rust/src/server/src/routes/openai/chat_completions/convert.rs index 04ae791bc3..2b3e3ddb36 100644 --- a/rust/src/server/src/routes/openai/chat_completions/convert.rs +++ b/rust/src/server/src/routes/openai/chat_completions/convert.rs @@ -18,11 +18,19 @@ use crate::utils::{ResolvedRequestContext, convert_logit_bias, merge_kv_transfer /// Lowered chat request plus the public response metadata carried by every SSE /// chunk. #[derive(Debug, Clone, PartialEq)] -pub struct PreparedRequest { +pub(super) struct PreparedRequest { /// Stable OpenAI-style request ID, reused as the external chat request ID. pub request_id: String, /// Public model ID echoed back to the client. pub response_model: String, + /// Public response rendering options for route-layer helpers. + pub options: ResponseOptions, + /// Lowered chat request for `vllm-chat`. + pub chat_request: ChatRequest, +} + +#[derive(Debug, Clone, Default, PartialEq)] +pub(super) struct ResponseOptions { /// Whether the caller asked for the final streamed usage chunk. pub include_usage: bool, /// Whether the caller requested output logprobs on chat choices. @@ -31,8 +39,6 @@ pub struct PreparedRequest { pub include_prompt_logprobs: bool, /// Whether to include reasoning content in OpenAI responses. pub include_reasoning: bool, - /// Lowered chat request for `vllm-chat`. - pub chat_request: ChatRequest, /// Last assistant-role message content to echo back when `echo=true`. pub echo: Option, /// Whether to include token IDs alongside generated text. @@ -46,7 +52,7 @@ pub struct PreparedRequest { /// /// `lora_resolution.model_names` must be non-empty; the first entry is used as /// the base `model` field in responses when no LoRA adapter is selected. -pub(crate) fn prepare_chat_request( +pub(super) fn prepare_chat_request( request: ChatCompletionRequest, lora_resolution: &LoraModelResolution, ctx: ResolvedRequestContext, @@ -146,14 +152,16 @@ pub(crate) fn prepare_chat_request( Ok(PreparedRequest { request_id, response_model, - include_usage, - requested_logprobs, - include_prompt_logprobs, - include_reasoning, + options: ResponseOptions { + include_usage, + requested_logprobs, + include_prompt_logprobs, + include_reasoning, + echo, + return_token_ids: request.return_token_ids.unwrap_or(false), + return_tokens_as_token_ids: request.return_tokens_as_token_ids.unwrap_or(false), + }, chat_request, - echo, - return_token_ids: request.return_token_ids.unwrap_or(false), - return_tokens_as_token_ids: request.return_tokens_as_token_ids.unwrap_or(false), }) } pub(crate) fn normalize_generation_prompt_mode( @@ -497,7 +505,7 @@ mod tests { ) .expect("request is valid"); - assert!(!prepared.include_reasoning); + assert!(!prepared.options.include_reasoning); } #[test] @@ -867,8 +875,8 @@ mod tests { ) .expect("request is valid"); - assert!(prepared.requested_logprobs); - assert!(prepared.include_prompt_logprobs); + assert!(prepared.options.requested_logprobs); + assert!(prepared.options.include_prompt_logprobs); assert_eq!(prepared.chat_request.sampling_params.logprobs, Some(0)); assert_eq!( prepared.chat_request.sampling_params.prompt_logprobs, @@ -894,7 +902,7 @@ mod tests { assert_eq!(prepared.chat_request.sampling_params.logprobs, Some(3)); assert_eq!(prepared.chat_request.sampling_params.prompt_logprobs, None); - assert!(!prepared.include_prompt_logprobs); + assert!(!prepared.options.include_prompt_logprobs); } #[test] diff --git a/rust/src/server/src/routes/openai/completions.rs b/rust/src/server/src/routes/openai/completions.rs index 9eda8b9d2a..8e3c300997 100644 --- a/rust/src/server/src/routes/openai/completions.rs +++ b/rust/src/server/src/routes/openai/completions.rs @@ -18,13 +18,13 @@ use tracing::{debug, error, info, trace}; use tracing_futures::Instrument as _; use vllm_text::{DecodedTextEvent, FinishReason, TextOutputStream, TextOutputStreamExt as _}; +use self::convert::{ResponseOptions, prepare_completion_request}; use super::utils::logprobs::{ collected_logprobs_to_openai, decoded_logprobs_to_openai, decoded_prompt_logprobs_to_maps, text_len, }; use super::utils::types::Usage; use crate::error::{ApiError, bail_server_error, server_error}; -use crate::routes::openai::completions::convert::prepare_completion_request; use crate::routes::openai::completions::types::{ CompletionChoice, CompletionRequest, CompletionResponse, CompletionSseChunk, CompletionStreamChoice, CompletionStreamResponse, @@ -42,7 +42,6 @@ pub async fn completions( ValidatedJson(body): ValidatedJson, ) -> Response { let stream = body.stream; - let logprobs = body.logprobs; let request_context = resolve_request_context(&headers, body.request_id.as_deref()); let lora_resolution = state.resolve_model_with_loras(Some(&body.model)).await; @@ -57,9 +56,7 @@ pub async fn completions( ); let created = unix_timestamp(); - let include_prompt_logprobs = prepared.text_request.sampling_params.prompt_logprobs.is_some(); let log_request = state.enable_log_requests; - let text_stream = match state .chat .text() @@ -84,11 +81,7 @@ pub async fn completions( prepared.response_model, created, log_request, - prepared.include_usage, - prepared.echo, - logprobs, - prepared.return_token_ids, - prepared.return_tokens_as_token_ids, + prepared.options, ); let sse_stream = completion_sse_stream(chunk_stream).instrument(request_span); @@ -99,11 +92,8 @@ pub async fn completions( prepared.request_id, prepared.response_model, created, - prepared.echo, - logprobs, - include_prompt_logprobs, - prepared.return_token_ids, - prepared.return_tokens_as_token_ids, + log_request, + prepared.options, ) .instrument(request_span.clone()) .await @@ -112,18 +102,6 @@ pub async fn completions( Err(error) => return error.into_response(), }; - if log_request { - let usage = response.usage.as_ref(); - info!( - parent: &request_span, - model = %response.model, - prompt_tokens = usage.map_or(0, |u| u.prompt_tokens), - output_tokens = usage.and_then(|u| u.completion_tokens).unwrap_or(0), - finish_reason = response.choices.first().and_then(|c| c.finish_reason.as_deref()).unwrap_or("unknown"), - "completion finished" - ); - } - Json(response).into_response() } } @@ -133,11 +111,16 @@ async fn collect_completion( request_id: String, response_model: String, created: u64, - echo: Option, - requested_logprobs: Option, - include_prompt_logprobs: bool, - return_token_ids: bool, - return_tokens_as_token_ids: bool, + log_request: bool, + ResponseOptions { + // Ignored: non-streaming responses always include usage. + include_usage: _, + echo, + requested_logprobs, + include_prompt_logprobs, + return_token_ids, + return_tokens_as_token_ids, + }: ResponseOptions, ) -> Result { let collected = stream .collect_output() @@ -175,6 +158,21 @@ async fn collect_completion( None => collected.text, Some(prompt) => format!("{prompt}{}", collected.text), }; + let finish_reason = completion_finish_reason_to_openai(finish_reason)?.to_string(); + let usage = Usage::from_counts( + collected.prompt_token_ids.len() as u32, + collected.token_ids.len() as u32, + ); + + if log_request { + info!( + model = %response_model, + prompt_tokens = usage.prompt_tokens, + output_tokens = usage.completion_tokens.unwrap_or(0), + %finish_reason, + "completion finished" + ); + } Ok(CompletionResponse { id: request_id, @@ -185,16 +183,13 @@ async fn collect_completion( index: 0, text, logprobs, - finish_reason: Some(completion_finish_reason_to_openai(finish_reason)?.into()), + finish_reason: Some(finish_reason), stop_reason, prompt_logprobs, token_ids: return_token_ids.then(|| collected.token_ids.clone()), prompt_token_ids: return_token_ids.then(|| collected.prompt_token_ids.to_vec()), }], - usage: Some(Usage::from_counts( - collected.prompt_token_ids.len() as u32, - collected.token_ids.len() as u32, - )), + usage: Some(usage), system_fingerprint: None, kv_transfer_params: collected.kv_transfer_params, }) @@ -208,11 +203,15 @@ async fn completion_chunk_stream( response_model: String, created: u64, log_request: bool, - include_usage: bool, - echo: Option, - requested_logprobs: Option, - return_token_ids: bool, - return_tokens_as_token_ids: bool, + ResponseOptions { + include_usage, + echo, + requested_logprobs, + // Ignored: streaming prompt logprobs are rejected for Python parity. + include_prompt_logprobs: _, + return_token_ids, + return_tokens_as_token_ids, + }: ResponseOptions, mut y: TryYielder, ) -> Result<(), ApiError> { pin_mut!(stream); @@ -432,7 +431,7 @@ mod tests { FinishReason, Finished, }; - use super::{CompletionSseChunk, completion_chunk_stream, final_chunk}; + use super::{CompletionSseChunk, ResponseOptions, completion_chunk_stream, final_chunk}; #[test] fn final_chunk_maps_stop_finish_reason() { @@ -527,11 +526,10 @@ mod tests { "model".to_string(), 1, false, - false, - None, - Some(1), - false, - false, + ResponseOptions { + requested_logprobs: Some(1), + ..Default::default() + }, ) .collect::>() .await; diff --git a/rust/src/server/src/routes/openai/completions/convert.rs b/rust/src/server/src/routes/openai/completions/convert.rs index 2d4ff08939..1dd73a4f53 100644 --- a/rust/src/server/src/routes/openai/completions/convert.rs +++ b/rust/src/server/src/routes/openai/completions/convert.rs @@ -10,18 +10,28 @@ use crate::utils::{ResolvedRequestContext, convert_logit_bias, merge_kv_transfer /// Lowered completion request plus the public response metadata carried by /// every SSE chunk. #[derive(Debug, Clone, PartialEq)] -pub struct PreparedRequest { +pub(super) struct PreparedRequest { /// Stable OpenAI-style request ID, reused as the external text request ID. pub request_id: String, /// Public model ID echoed back to the client. pub response_model: String, - /// Whether the caller asked for the final streamed usage chunk. - pub include_usage: bool, + /// Public response rendering options for route-layer helpers. + pub options: ResponseOptions, /// Lowered text request for the shared `vllm-text` facade. pub text_request: TextRequest, +} + +#[derive(Debug, Clone, Default, PartialEq)] +pub(super) struct ResponseOptions { + /// Whether the caller asked for the final streamed usage chunk. + pub include_usage: bool, /// Original text prompt that should be echoed back northbound when /// `echo=true`. pub echo: Option, + /// Whether the caller requested output logprobs on completion choices. + pub requested_logprobs: Option, + /// Whether the caller requested choice-level prompt logprobs. + pub include_prompt_logprobs: bool, /// Whether to include token IDs alongside generated text. pub return_token_ids: bool, /// Whether to format logprob tokens as `token_id:{id}`. @@ -33,7 +43,7 @@ pub struct PreparedRequest { /// /// `lora_resolution.model_names` must be non-empty; the first entry is used as /// the base `model` field in responses when no LoRA adapter is selected. -pub(crate) fn prepare_completion_request( +pub(super) fn prepare_completion_request( request: CompletionRequest, lora_resolution: &LoraModelResolution, ctx: ResolvedRequestContext, @@ -64,6 +74,7 @@ pub(crate) fn prepare_completion_request( let include_usage = (request.stream_options.as_ref()) .and_then(|options| options.include_usage) .unwrap_or(false); + let include_prompt_logprobs = prompt_logprobs.is_some(); let echo = request.echo.then(|| request.prompt.as_text().cloned()).flatten(); let structured_outputs = @@ -116,11 +127,15 @@ pub(crate) fn prepare_completion_request( Ok(PreparedRequest { request_id, response_model, - include_usage, + options: ResponseOptions { + include_usage, + echo, + requested_logprobs: request.logprobs, + include_prompt_logprobs, + return_token_ids: request.return_token_ids.unwrap_or(false), + return_tokens_as_token_ids: request.return_tokens_as_token_ids.unwrap_or(false), + }, text_request, - echo, - return_token_ids: request.return_token_ids.unwrap_or(false), - return_tokens_as_token_ids: request.return_tokens_as_token_ids.unwrap_or(false), }) } @@ -206,7 +221,7 @@ mod tests { ) .expect("prepare"); - assert!(prepared.include_usage); + assert!(prepared.options.include_usage); assert_eq!( prepared.text_request.prompt, Prompt::TokenIds(vec![11, 22, 33]) @@ -250,7 +265,7 @@ mod tests { ) .expect("prepare"); - assert_eq!(prepared.echo, Some("hello".to_string())); + assert_eq!(prepared.options.echo, Some("hello".to_string())); assert_eq!(prepared.text_request.sampling_params.max_tokens, Some(7)); } diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 67aa9cccff..838fee5c28 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -14,15 +14,14 @@ use std::{fmt, fs}; use axum::body::{Body, to_bytes}; use axum::http::{Request, StatusCode}; use bytes::Bytes; -use futures::StreamExt as _; use rmpv::Value; use serde_json::json; use serial_test::serial; use tower::{Service as _, ServiceExt as _}; use vllm_chat::{ - ChatBackend, ChatContent, ChatContentPart, ChatEvent, ChatLlm, ChatMessage, ChatRenderer, - ChatRequest, ChatRole, ChatTextBackend, DefaultChatOutputProcessor, DynChatOutputProcessor, - DynChatRenderer, NewChatOutputProcessorOptions, SamplingParams, + ChatBackend, ChatContent, ChatContentPart, ChatLlm, ChatMessage, ChatRenderer, ChatRequest, + ChatTextBackend, DefaultChatOutputProcessor, DynChatOutputProcessor, DynChatRenderer, + NewChatOutputProcessorOptions, }; use vllm_engine_core_client::protocol::logprobs::{ Logprobs, MaybeWireLogprobs, PositionLogprobs, TokenLogprob, @@ -44,8 +43,6 @@ use zeromq::prelude::{SocketRecv, SocketSend}; use zeromq::{DealerSocket, PushSocket, ZmqMessage}; use super::{build_router, build_router_with_dev_mode, build_router_with_dev_mode_and_lora}; -use crate::lora::LoraModelResolution; -use crate::routes::openai::chat_completions::convert::prepare_chat_request; use crate::state::AppState; fn request_output( @@ -3478,92 +3475,6 @@ async fn completions_echo_stream_emits_separate_prompt_chunk() { assert_eq!(usage_chunk["usage"]["completion_tokens"], 3); } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[serial] -async fn chat_harness_streams_text_events() { - let (chat, engine_task) = test_chat_with_engine_handle().await; - let mut stream = chat - .chat(ChatRequest { - messages: vec![ChatMessage::text(ChatRole::User, "hello")], - sampling_params: SamplingParams { - max_tokens: Some(8), - ..Default::default() - }, - request_id: "chat-harness".to_string(), - ..ChatRequest::for_test() - }) - .await - .expect("submit chat request"); - - let mut saw_text = false; - let mut saw_done = false; - while let Some(event) = stream.next().await { - match event.expect("chat event") { - ChatEvent::BlockDelta { .. } => saw_text = true, - ChatEvent::Done { .. } => { - saw_done = true; - break; - } - ChatEvent::Start { .. } - | ChatEvent::LogprobsDelta { .. } - | ChatEvent::BlockStart { .. } - | ChatEvent::BlockEnd { .. } - | ChatEvent::ToolCallStart { .. } - | ChatEvent::ToolCallArgumentsDelta { .. } - | ChatEvent::ToolCallEnd { .. } => {} - } - } - engine_task.await.expect("mock engine task"); - - assert!(saw_text); - assert!(saw_done); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[serial] -async fn prepared_openai_request_streams_text_events() { - let (chat, engine_task) = test_chat_with_engine_handle().await; - let prepared = prepare_chat_request( - serde_json::from_value(json!({ - "model": "Qwen/Qwen1.5-0.5B-Chat", - "stream": true, - "messages": [{"role": "user", "content": "hello"}] - })) - .expect("decode request"), - &LoraModelResolution { - model_names: vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()], - lora_request: None, - }, - crate::utils::ResolvedRequestContext::default(), - ) - .expect("prepare request"); - - let mut stream = chat.chat(prepared.chat_request).await.expect("submit chat request"); - - let mut saw_text = false; - let mut saw_done = false; - while let Some(event) = stream.next().await { - match event.expect("chat event") { - ChatEvent::BlockDelta { .. } => saw_text = true, - ChatEvent::Done { .. } => { - saw_done = true; - break; - } - ChatEvent::Start { .. } - | ChatEvent::LogprobsDelta { .. } - | ChatEvent::BlockStart { .. } - | ChatEvent::BlockEnd { .. } - | ChatEvent::ToolCallStart { .. } - | ChatEvent::ToolCallArgumentsDelta { .. } - | ChatEvent::ToolCallEnd { .. } => {} - } - } - engine_task.await.expect("mock engine task"); - - assert!(saw_text); - assert!(saw_done); -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn reasoning_blocks_are_mapped_to_reasoning_sse_chunks() { diff --git a/rust/src/server/src/state.rs b/rust/src/server/src/state.rs index 0dd3047d0c..bcb5f1c6d9 100644 --- a/rust/src/server/src/state.rs +++ b/rust/src/server/src/state.rs @@ -10,7 +10,6 @@ use vllm_engine_core_client::EngineCoreClient; use vllm_engine_core_client::protocol::lora::LoraRequest; use crate::lora::{LoadLoraError, LoraManager, LoraModelResolution, UnloadLoraError}; - use crate::server_info::{ServerInfoConfigFormat, ServerInfoSnapshot}; const SHUTDOWN_REFCOUNT_POLL_INTERVAL: Duration = Duration::from_millis(100);