diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 3725ebd3c4..fea8c3841a 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -90,7 +90,7 @@ rlimit = "0.11.0" rmp-serde = "1.3.1" rmpv = { version = "1.3.1", features = ["with-serde"] } rustc-hash = "1.1.0" -serde = { version = "1.0.228", features = ["derive"] } +serde = { version = "1.0.228", features = ["derive", "rc"] } serde-json-fmt = "0.1.0" serde_default = "0.2.0" serde_json = { version = "1.0.145", features = ["preserve_order"] } diff --git a/rust/src/server/src/routes/inference/generate.rs b/rust/src/server/src/routes/inference/generate.rs index aa84913f23..308ec3ca3f 100644 --- a/rust/src/server/src/routes/inference/generate.rs +++ b/rust/src/server/src/routes/inference/generate.rs @@ -395,16 +395,18 @@ async fn generate_sse_stream( } fn to_sse_event(chunk: &GenerateStreamResponse) -> Event { - let payload = serde_json::to_string(chunk).expect("generate chunk must serialize to JSON"); - trace!(payload, "generate emitting chunk"); - Event::default().data(payload) + trace!(?chunk, "generate emitting chunk"); + Event::default() + .json_data(chunk) + .expect("generate chunk must serialize to JSON") } fn to_error_sse_event(error: &ApiError) -> Event { - let payload = serde_json::to_string(&error.to_error_response()) - .expect("ErrorResponse must serialize to JSON"); - trace!(payload, "generate emitting error"); - Event::default().data(payload) + let response = error.to_error_response(); + trace!(?response, "generate emitting error"); + Event::default() + .json_data(response) + .expect("ErrorResponse must serialize to JSON") } fn done_sse_event() -> Event { diff --git a/rust/src/server/src/routes/openai/chat_completions.rs b/rust/src/server/src/routes/openai/chat_completions.rs index 0485192191..40934e6095 100644 --- a/rust/src/server/src/routes/openai/chat_completions.rs +++ b/rust/src/server/src/routes/openai/chat_completions.rs @@ -39,7 +39,8 @@ use crate::routes::openai::utils::logprobs::{ decoded_logprobs_to_openai_chat, prompt_logprobs_to_maps, }; use crate::routes::openai::utils::types::{ - ChatLogProbs, FunctionCallDelta, FunctionCallResponse, ToolCall, ToolCallDelta, Usage, + ChatLogProbs, FunctionCallDelta, FunctionCallResponse, StreamResponseEnvelope, ToolCall, + ToolCallDelta, Usage, }; use crate::routes::openai::utils::usage::ContinuousUsage; use crate::routes::openai::utils::validated_json::ValidatedJson; @@ -270,6 +271,12 @@ async fn chat_completion_chunk_stream( }: ResponseOptions, mut y: TryYielder, ) -> Result<(), ApiError> { + let envelope = Arc::new(StreamResponseEnvelope::new( + request_id, + "chat.completion.chunk", + created, + response_model, + )); let mut saw_tool_calls = false; // `LogprobsDelta` is emitted after all chat events for one decoded update. // If that update contains hidden reasoning, including delimiter-only block @@ -303,7 +310,7 @@ async fn chat_completion_chunk_stream( prompt_token_ids, .. }) => { continuous_usage.set_prompt_tokens(prompt_token_ids.len()); - let mut chunk = start_chunk(&request_id, &response_model, created); + let mut chunk = start_chunk(&envelope); if return_token_ids { chunk.prompt_token_ids = Some(prompt_token_ids.to_vec()); } @@ -311,9 +318,7 @@ async fn chat_completion_chunk_stream( // When echo=true, emit the last assistant message content as a delta chunk. if let Some(echo_text) = &echo { yield_chunk!(block_delta_chunk( - &request_id, - &response_model, - created, + &envelope, AssistantBlockKind::Text, echo_text.clone(), )); @@ -326,13 +331,7 @@ async fn chat_completion_chunk_stream( if let Some(pending_chunk) = pending_chunk.as_mut() { pending_chunk.push_block_delta(kind, delta); } else { - yield_chunk!(block_delta_chunk( - &request_id, - &response_model, - created, - kind, - delta, - )); + yield_chunk!(block_delta_chunk(&envelope, kind, delta)); } } else { suppress_current_update_metadata = true; @@ -368,18 +367,11 @@ async fn chat_completion_chunk_stream( if let Some(pending_chunk) = pending_chunk.as_mut() { pending_chunk.logprobs = openai_logprobs; pending_chunk.token_ids = openai_token_ids; - if let Some(chunk) = - pending_chunk.take_chunk(&request_id, &response_model, created) - { + if let Some(chunk) = pending_chunk.take_chunk(&envelope) { yield_chunk!(chunk); } } else if let Some(logprobs) = openai_logprobs { - yield_chunk!(logprobs_only_chunk( - &request_id, - &response_model, - created, - logprobs, - )); + yield_chunk!(logprobs_only_chunk(&envelope, logprobs)); } } Ok(ChatEvent::BlockStart { kind, .. }) => { @@ -407,14 +399,7 @@ async fn chat_completion_chunk_stream( if let Some(pending_chunk) = pending_chunk.as_mut() { pending_chunk.push_tool_call_start(tool_index, id, name); } else { - yield_chunk!(tool_call_start_chunk( - &request_id, - &response_model, - created, - tool_index, - id, - name, - )); + yield_chunk!(tool_call_start_chunk(&envelope, tool_index, id, name)); } } Ok(ChatEvent::ToolCallArgumentsDelta { index, delta }) => { @@ -422,13 +407,7 @@ async fn chat_completion_chunk_stream( if let Some(pending_chunk) = pending_chunk.as_mut() { pending_chunk.push_tool_call_arguments(tool_index, delta); } else { - yield_chunk!(tool_call_arguments_chunk( - &request_id, - &response_model, - created, - tool_index, - delta, - )); + yield_chunk!(tool_call_arguments_chunk(&envelope, tool_index, delta)); } } Ok(ChatEvent::ToolCallEnd { .. }) => { @@ -442,7 +421,7 @@ async fn chat_completion_chunk_stream( if enable_log_requests { info!( stream = true, - model = %response_model, + model = %envelope.model(), prompt_tokens = final_usage.prompt_token_count, output_tokens = final_usage.output_token_count, finish_reason = finish_reason.as_str(), @@ -456,16 +435,13 @@ async fn chat_completion_chunk_stream( ); if let Some(pending_chunk) = pending_chunk.as_mut() - && let Some(chunk) = - pending_chunk.take_chunk(&request_id, &response_model, created) + && let Some(chunk) = pending_chunk.take_chunk(&envelope) { yield_chunk!(chunk); } match final_chunk( - &request_id, - &response_model, - created, + &envelope, finish_reason, saw_tool_calls && !is_named_tool_choice, ) { @@ -481,9 +457,7 @@ async fn chat_completion_chunk_stream( if include_usage { y.yield_ok(usage_chunk( - &request_id, - &response_model, - created, + &envelope, Usage::from_token_usage(final_usage, enable_prompt_tokens_details), )) .await; @@ -504,12 +478,10 @@ async fn chat_completion_chunk_stream( } fn usage_chunk( - request_id: &str, - response_model: &str, - created: u64, + envelope: &Arc, usage: Usage, ) -> ChatCompletionStreamResponse { - let mut chunk = ChatCompletionStreamResponse::new(request_id, response_model, created); + let mut chunk = ChatCompletionStreamResponse::new(envelope); chunk.usage = Some(usage); chunk } @@ -588,9 +560,7 @@ impl PendingChatChunk { /// necessarily with a visible/chat-semantic delta. fn take_chunk( &mut self, - request_id: &str, - response_model: &str, - created: u64, + envelope: &Arc, ) -> Option { let has_delta = self.delta.content.is_some() || self.delta.reasoning.is_some() @@ -601,7 +571,7 @@ impl PendingChatChunk { return None; } - let mut chunk = ChatCompletionStreamResponse::new(request_id, response_model, created); + let mut chunk = ChatCompletionStreamResponse::new(envelope); chunk.choices.push(ChatCompletionStreamChoice { delta: self.take_delta(), logprobs, @@ -659,18 +629,19 @@ async fn chat_completion_sse_stream( /// Serialize one OpenAI chunk payload into one SSE `data:` event. fn to_sse_event(chunk: &ChatCompletionStreamResponse) -> Event { - let payload = - serde_json::to_string(chunk).expect("ChatCompletionStreamResponse must serialize to JSON"); - trace!(payload, "chat completion emitting chunk"); - Event::default().data(payload) + trace!(?chunk, "chat completion emitting chunk"); + Event::default() + .json_data(chunk) + .expect("ChatCompletionStreamResponse must serialize to JSON") } /// Serialize one OpenAI error payload into one SSE `data:` event. fn to_error_sse_event(error: &ApiError) -> Event { - let payload = serde_json::to_string(&error.to_error_response()) - .expect("ErrorResponse must serialize to JSON"); - trace!(payload, "chat completion emitting error"); - Event::default().data(payload) + let response = error.to_error_response(); + trace!(?response, "chat completion emitting error"); + Event::default() + .json_data(response) + .expect("ErrorResponse must serialize to JSON") } /// Build the terminal OpenAI SSE sentinel event. @@ -681,12 +652,8 @@ fn done_sse_event() -> Event { /// Build the initial assistant-role SSE chunk required by the OpenAI streaming /// protocol. -fn start_chunk( - request_id: &str, - response_model: &str, - created: u64, -) -> ChatCompletionStreamResponse { - let mut chunk = ChatCompletionStreamResponse::new(request_id, response_model, created); +fn start_chunk(envelope: &Arc) -> ChatCompletionStreamResponse { + let mut chunk = ChatCompletionStreamResponse::new(envelope); chunk.choices.push(ChatCompletionStreamChoice { delta: ChatMessageDelta { role: Some(AssistantRole), @@ -699,9 +666,7 @@ fn start_chunk( /// Build one content-delta SSE chunk from one internal assistant block delta. fn block_delta_chunk( - request_id: &str, - response_model: &str, - created: u64, + envelope: &Arc, kind: AssistantBlockKind, delta: String, ) -> ChatCompletionStreamResponse { @@ -719,7 +684,7 @@ fn block_delta_chunk( } }; - let mut chunk = ChatCompletionStreamResponse::new(request_id, response_model, created); + let mut chunk = ChatCompletionStreamResponse::new(envelope); chunk.choices.push(ChatCompletionStreamChoice { delta, ..Default::default() @@ -728,14 +693,12 @@ fn block_delta_chunk( } fn tool_call_start_chunk( - request_id: &str, - response_model: &str, - created: u64, + envelope: &Arc, tool_index: u32, id: String, name: String, ) -> ChatCompletionStreamResponse { - let mut chunk = ChatCompletionStreamResponse::new(request_id, response_model, created); + let mut chunk = ChatCompletionStreamResponse::new(envelope); chunk.choices.push(ChatCompletionStreamChoice { delta: ChatMessageDelta { tool_calls: Some(vec![ToolCallDelta { @@ -755,13 +718,11 @@ fn tool_call_start_chunk( } fn tool_call_arguments_chunk( - request_id: &str, - response_model: &str, - created: u64, + envelope: &Arc, tool_index: u32, delta: String, ) -> ChatCompletionStreamResponse { - let mut chunk = ChatCompletionStreamResponse::new(request_id, response_model, created); + let mut chunk = ChatCompletionStreamResponse::new(envelope); chunk.choices.push(ChatCompletionStreamChoice { delta: ChatMessageDelta { tool_calls: Some(vec![ToolCallDelta { @@ -781,12 +742,10 @@ fn tool_call_arguments_chunk( } fn logprobs_only_chunk( - request_id: &str, - response_model: &str, - created: u64, + envelope: &Arc, logprobs: ChatLogProbs, ) -> ChatCompletionStreamResponse { - let mut chunk = ChatCompletionStreamResponse::new(request_id, response_model, created); + let mut chunk = ChatCompletionStreamResponse::new(envelope); chunk.choices.push(ChatCompletionStreamChoice { logprobs: Some(logprobs), ..Default::default() @@ -796,9 +755,7 @@ fn logprobs_only_chunk( /// Build the terminal SSE chunk carrying the OpenAI finish reason. fn final_chunk( - request_id: &str, - response_model: &str, - created: u64, + envelope: &Arc, finish_reason: FinishReason, use_tool_calls_finish_reason: bool, ) -> Result { @@ -811,7 +768,7 @@ fn final_chunk( "chat stream finished" ); - let mut chunk = ChatCompletionStreamResponse::new(request_id, response_model, created); + let mut chunk = ChatCompletionStreamResponse::new(envelope); chunk.choices.push(ChatCompletionStreamChoice { finish_reason: Some(finish_reason.to_string()), stop_reason, @@ -844,6 +801,8 @@ fn stop_reason_to_json(stop_reason: &StopReason) -> Value { #[cfg(test)] mod tests { + use std::sync::Arc; + use futures::{StreamExt as _, stream}; use serde_json::json; use vllm_chat::{ @@ -853,19 +812,23 @@ mod tests { use vllm_text::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTokenLogprob}; use super::{ - ApiServerOptions, ResponseOptions, block_delta_chunk, chat_completion_chunk_stream, - final_chunk, + ApiServerOptions, ResponseOptions, StreamResponseEnvelope, block_delta_chunk, + chat_completion_chunk_stream, final_chunk, }; + fn stream_envelope() -> Arc { + Arc::new(StreamResponseEnvelope::new( + "chatcmpl-1".to_string(), + "chat.completion.chunk", + 1, + "model".to_string(), + )) + } + #[test] fn text_chunk_uses_content_only_delta() { - let chunk = block_delta_chunk( - "chatcmpl-1", - "model", - 1, - AssistantBlockKind::Text, - "hello".to_string(), - ); + let envelope = stream_envelope(); + let chunk = block_delta_chunk(&envelope, AssistantBlockKind::Text, "hello".to_string()); assert_eq!(chunk.choices[0].delta.role, None); assert_eq!(chunk.choices[0].delta.content.as_deref(), Some("hello")); assert_eq!(chunk.choices[0].delta.reasoning, None); @@ -873,10 +836,9 @@ mod tests { #[test] fn reasoning_chunk_uses_reasoning_only_delta() { + let envelope = stream_envelope(); let chunk = block_delta_chunk( - "chatcmpl-1", - "model", - 1, + &envelope, AssistantBlockKind::Reasoning, "thinking".to_string(), ); @@ -890,10 +852,9 @@ mod tests { #[test] fn final_chunk_maps_stop_finish_reason_and_stop_reason() { + let envelope = stream_envelope(); let chunk = final_chunk( - "chatcmpl-1", - "model", - 1, + &envelope, FinishReason::Stop(Some(StopReason::Text("stop".to_string()))), false, ) @@ -905,7 +866,7 @@ mod tests { #[test] fn final_chunk_maps_length_finish_reason() { - let chunk = final_chunk("chatcmpl-1", "model", 1, FinishReason::Length, false) + let chunk = final_chunk(&stream_envelope(), FinishReason::Length, false) .expect("finish reason is valid"); assert_eq!(chunk.choices[0].finish_reason.as_deref(), Some("length")); @@ -914,7 +875,7 @@ mod tests { #[test] fn final_chunk_maps_abort_finish_reason() { - let chunk = final_chunk("chatcmpl-1", "model", 1, FinishReason::Abort, false) + let chunk = final_chunk(&stream_envelope(), FinishReason::Abort, false) .expect("abort is a valid finish reason"); assert_eq!(chunk.choices[0].finish_reason.as_deref(), Some("abort")); @@ -923,12 +884,12 @@ mod tests { #[test] fn final_chunk_rejects_error_finish_reason() { - assert!(final_chunk("chatcmpl-1", "model", 1, FinishReason::Error, false).is_err()); + assert!(final_chunk(&stream_envelope(), FinishReason::Error, false).is_err()); } #[test] fn final_chunk_maps_stop_to_tool_calls_when_tool_calls_were_streamed() { - let chunk = final_chunk("chatcmpl-1", "model", 1, FinishReason::stop_eos(), true) + let chunk = final_chunk(&stream_envelope(), FinishReason::stop_eos(), true) .expect("finish reason is valid"); assert_eq!( @@ -1002,6 +963,7 @@ mod tests { .expect("stream chunks"); assert_eq!(chunks.len(), 4); + assert!(chunks.windows(2).all(|pair| Arc::ptr_eq(&pair[0].envelope, &pair[1].envelope))); assert_eq!(chunks[1].choices[0].delta.content.as_deref(), Some("hi")); let logprobs = chunks[1].choices[0].logprobs.as_ref().expect("logprobs"); let content = logprobs.content.as_ref().expect("logprobs content"); diff --git a/rust/src/server/src/routes/openai/chat_completions/types.rs b/rust/src/server/src/routes/openai/chat_completions/types.rs index 5b4e733d63..fc3c15021d 100644 --- a/rust/src/server/src/routes/openai/chat_completions/types.rs +++ b/rust/src/server/src/routes/openai/chat_completions/types.rs @@ -3,6 +3,7 @@ use std::collections::HashMap; use std::fmt; +use std::sync::Arc; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -13,9 +14,9 @@ use vllm_engine_core_client::protocol::sampling::RepetitionDetectionParams; use crate::routes::openai::utils::structured_outputs::ResponseFormat; use crate::routes::openai::utils::types::{ - ChatLogProbs, ChatMessage, Normalizable, PromptLogprobs, StreamOptions, StringOrArray, Tool, - ToolCall, ToolCallDelta, ToolChoice, Usage, default_true, deserialize_request_top_k, - validate_messages, validate_stop, validate_top_p_value, + ChatLogProbs, ChatMessage, Normalizable, PromptLogprobs, StreamOptions, StreamResponseEnvelope, + StringOrArray, Tool, ToolCall, ToolCallDelta, ToolChoice, Usage, default_true, + deserialize_request_top_k, validate_messages, validate_stop, validate_top_p_value, }; /// vLLM-compatible request type for the Chat Completions API. @@ -383,10 +384,8 @@ pub(super) struct ChatCompletionMessage { #[serde_with::skip_serializing_none] #[derive(Debug, Clone, Serialize)] pub(super) struct ChatCompletionStreamResponse { - pub id: String, - pub object: String, - pub created: u64, - pub model: String, + #[serde(flatten)] + pub envelope: Arc, pub choices: Vec, pub usage: Option, pub prompt_token_ids: Option>, @@ -394,12 +393,9 @@ pub(super) struct ChatCompletionStreamResponse { impl ChatCompletionStreamResponse { /// Create a stream response with the standard envelope fields pre-filled. - pub fn new(id: &str, model: &str, created: u64) -> Self { + pub fn new(envelope: &Arc) -> Self { Self { - id: id.to_string(), - object: "chat.completion.chunk".to_string(), - created, - model: model.to_string(), + envelope: Arc::clone(envelope), choices: Vec::new(), usage: None, prompt_token_ids: None, diff --git a/rust/src/server/src/routes/openai/completions.rs b/rust/src/server/src/routes/openai/completions.rs index d8943b69f3..ecb355299b 100644 --- a/rust/src/server/src/routes/openai/completions.rs +++ b/rust/src/server/src/routes/openai/completions.rs @@ -33,7 +33,7 @@ use super::utils::logprobs::{ collected_logprobs_to_openai, decoded_logprobs_to_openai, decoded_prompt_logprobs_to_openai, prompt_logprobs_to_maps, text_len, }; -use super::utils::types::Usage; +use super::utils::types::{StreamResponseEnvelope, Usage}; use crate::config::ApiServerOptions; use crate::error::{ApiError, bail_server_error, server_error, text_submit_error}; use crate::lora::LoraModelResolution; @@ -257,6 +257,12 @@ async fn completion_chunk_stream( mut y: TryYielder, ) -> Result<(), ApiError> { pin_mut!(stream); + let envelope = Arc::new(StreamResponseEnvelope::new( + request_id, + "text_completion", + created, + response_model, + )); let mut visible_text_len = 0_u32; let mut first_chunk = true; let mut continuous_usage = ContinuousUsage::default(); @@ -292,13 +298,7 @@ async fn completion_chunk_stream( } else { None }; - let mut chunk = delta_chunk( - &request_id, - &response_model, - created, - prompt.clone(), - logprobs, - ); + let mut chunk = delta_chunk(&envelope, prompt.clone(), logprobs); if return_token_ids && first_chunk { if let Some(choice) = chunk.choices.first_mut() { choice.prompt_token_ids = Some(prompt_token_ids.to_vec()); @@ -308,8 +308,7 @@ async fn completion_chunk_stream( yield_chunk!(chunk); } else if return_token_ids { // Emit a chunk with prompt_token_ids in the first streaming response - let mut chunk = - delta_chunk(&request_id, &response_model, created, String::new(), None); + let mut chunk = delta_chunk(&envelope, String::new(), None); if let Some(choice) = chunk.choices.first_mut() { choice.prompt_token_ids = Some(prompt_token_ids.to_vec()); } @@ -331,7 +330,7 @@ async fn completion_chunk_stream( if enable_log_requests { info!( stream = true, - model = %response_model, + model = %envelope.model(), prompt_tokens = finished.usage.prompt_token_count, output_tokens = finished.usage.output_token_count, finish_reason = finished.finish_reason.as_str(), @@ -342,19 +341,12 @@ async fn completion_chunk_stream( finished.usage.prompt_token_count, finished.usage.output_token_count, ); - let final_chunk = final_chunk( - &request_id, - &response_model, - created, - finished.finish_reason, - )?; + let final_chunk = final_chunk(&envelope, finished.finish_reason)?; yield_chunk!(final_chunk); if include_usage { y.yield_ok(CompletionSseChunk::Usage(usage_chunk( - &request_id, - &response_model, - created, + &envelope, Usage::from_token_usage( finished.usage, enable_prompt_tokens_details, @@ -380,7 +372,7 @@ async fn completion_chunk_stream( } else { None }; - let mut chunk = delta_chunk(&request_id, &response_model, created, delta, logprobs); + let mut chunk = delta_chunk(&envelope, delta, logprobs); let delta_token_count = token_ids.len(); continuous_usage.add_output_tokens(delta_token_count); if return_token_ids && let Some(choice) = chunk.choices.first_mut() { @@ -393,7 +385,7 @@ async fn completion_chunk_stream( if enable_log_requests { info!( stream = true, - model = %response_model, + model = %envelope.model(), prompt_tokens = finished.usage.prompt_token_count, output_tokens = finished.usage.output_token_count, finish_reason = finished.finish_reason.as_str(), @@ -404,19 +396,12 @@ async fn completion_chunk_stream( finished.usage.prompt_token_count, finished.usage.output_token_count, ); - let final_chunk = final_chunk( - &request_id, - &response_model, - created, - finished.finish_reason, - )?; + let final_chunk = final_chunk(&envelope, finished.finish_reason)?; yield_chunk!(final_chunk); if include_usage { y.yield_ok(CompletionSseChunk::Usage(usage_chunk( - &request_id, - &response_model, - created, + &envelope, Usage::from_token_usage(finished.usage, enable_prompt_tokens_details), ))) .await; @@ -436,13 +421,11 @@ async fn completion_chunk_stream( } fn delta_chunk( - request_id: &str, - response_model: &str, - created: u64, + envelope: &Arc, text: String, logprobs: Option, ) -> CompletionStreamResponse { - let mut chunk = CompletionStreamResponse::new(request_id, response_model, created); + let mut chunk = CompletionStreamResponse::new(envelope); chunk.choices.push(CompletionStreamChoice { text, logprobs, @@ -452,15 +435,13 @@ fn delta_chunk( } fn final_chunk( - request_id: &str, - response_model: &str, - created: u64, + envelope: &Arc, finish_reason: FinishReason, ) -> Result { let stop_reason = finish_reason.as_stop_reason().map(stop_reason_to_json); let finish_reason = completion_finish_reason_to_openai(&finish_reason)?; - let mut chunk = CompletionStreamResponse::new(request_id, response_model, created); + let mut chunk = CompletionStreamResponse::new(envelope); chunk.choices.push(CompletionStreamChoice { finish_reason: Some(finish_reason.to_string()), stop_reason, @@ -517,13 +498,8 @@ fn prompt_only_logprobs_to_openai( )) } -fn usage_chunk( - request_id: &str, - response_model: &str, - created: u64, - usage: Usage, -) -> CompletionStreamResponse { - let mut chunk = CompletionStreamResponse::new(request_id, response_model, created); +fn usage_chunk(envelope: &Arc, usage: Usage) -> CompletionStreamResponse { + let mut chunk = CompletionStreamResponse::new(envelope); chunk.usage = Some(usage); chunk } @@ -556,17 +532,19 @@ async fn completion_sse_stream( /// Serialize one OpenAI chunk payload into one SSE `data:` event. fn to_sse_event(chunk: &CompletionSseChunk) -> Event { - let payload = serde_json::to_string(chunk).expect("completion chunk must serialize to JSON"); - trace!(payload, "completion emitting chunk"); - Event::default().data(payload) + trace!(?chunk, "completion emitting chunk"); + Event::default() + .json_data(chunk) + .expect("completion chunk must serialize to JSON") } /// Serialize one OpenAI error payload into one SSE `data:` event. fn to_error_sse_event(error: &ApiError) -> Event { - let payload = serde_json::to_string(&error.to_error_response()) - .expect("ErrorResponse must serialize to JSON"); - trace!(payload, "completion emitting error"); - Event::default().data(payload) + let response = error.to_error_response(); + trace!(?response, "completion emitting error"); + Event::default() + .json_data(response) + .expect("ErrorResponse must serialize to JSON") } /// Build the terminal OpenAI SSE sentinel event. @@ -577,6 +555,8 @@ fn done_sse_event() -> Event { #[cfg(test)] mod tests { + use std::sync::Arc; + use futures::{StreamExt as _, stream}; use itertools::Itertools as _; use vllm_engine_core_client::protocol::output::StopReason; @@ -586,13 +566,29 @@ mod tests { }; use super::{ - ApiServerOptions, CompletionSseChunk, ResponseOptions, completion_chunk_stream, final_chunk, + ApiServerOptions, CompletionSseChunk, CompletionStreamResponse, ResponseOptions, + StreamResponseEnvelope, completion_chunk_stream, final_chunk, }; + fn stream_envelope() -> Arc { + Arc::new(StreamResponseEnvelope::new( + "cmpl-1".to_string(), + "text_completion", + 1, + "model".to_string(), + )) + } + + fn chunk_response(chunk: &CompletionSseChunk) -> &CompletionStreamResponse { + match chunk { + CompletionSseChunk::Chunk(response) | CompletionSseChunk::Usage(response) => response, + } + } + #[test] fn final_chunk_maps_stop_finish_reason() { - let chunk = final_chunk("cmpl-1", "model", 1, FinishReason::stop_eos()) - .expect("finish reason valid"); + let chunk = + final_chunk(&stream_envelope(), FinishReason::stop_eos()).expect("finish reason valid"); assert_eq!(chunk.choices[0].finish_reason.as_deref(), Some("stop")); assert_eq!(chunk.choices[0].text, ""); } @@ -600,20 +596,20 @@ mod tests { #[test] fn final_chunk_maps_length_finish_reason() { let chunk = - final_chunk("cmpl-1", "model", 1, FinishReason::Length).expect("finish reason valid"); + final_chunk(&stream_envelope(), FinishReason::Length).expect("finish reason valid"); assert_eq!(chunk.choices[0].finish_reason.as_deref(), Some("length")); } #[test] fn final_chunk_maps_abort_finish_reason() { let chunk = - final_chunk("cmpl-1", "model", 1, FinishReason::Abort).expect("finish reason valid"); + final_chunk(&stream_envelope(), FinishReason::Abort).expect("finish reason valid"); assert_eq!(chunk.choices[0].finish_reason.as_deref(), Some("abort")); } #[test] fn final_chunk_rejects_error_finish_reason() { - assert!(final_chunk("cmpl-1", "model", 1, FinishReason::Error).is_err()); + assert!(final_chunk(&stream_envelope(), FinishReason::Error).is_err()); } #[tokio::test] @@ -702,6 +698,13 @@ mod tests { let chunks: Vec<_> = chunks.into_iter().try_collect().expect("stream should succeed"); + assert!(chunks.windows(2).all(|pair| { + Arc::ptr_eq( + &chunk_response(&pair[0]).envelope, + &chunk_response(&pair[1]).envelope, + ) + })); + match &chunks[0] { CompletionSseChunk::Chunk(chunk) => { assert_eq!(chunk.choices[0].text, "h"); diff --git a/rust/src/server/src/routes/openai/completions/types.rs b/rust/src/server/src/routes/openai/completions/types.rs index b83cf06717..92e6119beb 100644 --- a/rust/src/server/src/routes/openai/completions/types.rs +++ b/rust/src/server/src/routes/openai/completions/types.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright contributors to the vLLM project use std::collections::HashMap; +use std::sync::Arc; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; @@ -10,8 +11,8 @@ use vllm_engine_core_client::protocol::sampling::RepetitionDetectionParams; use vllm_text::Prompt; use crate::routes::openai::utils::types::{ - LogProbs, Normalizable, PromptLogprobs, StreamOptions, StringOrArray, Usage, default_true, - deserialize_request_top_k, validate_stop, + LogProbs, Normalizable, PromptLogprobs, StreamOptions, StreamResponseEnvelope, StringOrArray, + Usage, default_true, deserialize_request_top_k, validate_stop, }; /// Serde default for `CompletionRequest::max_tokens`, matching the Python vLLM @@ -241,22 +242,17 @@ pub(super) struct CompletionChoice { #[serde_with::skip_serializing_none] #[derive(Debug, Clone, Serialize)] pub(super) struct CompletionStreamResponse { - pub id: String, - pub object: String, - pub created: u64, - pub model: String, + #[serde(flatten)] + pub envelope: Arc, pub choices: Vec, pub usage: Option, } impl CompletionStreamResponse { /// Create a stream response with the standard envelope fields pre-filled. - pub fn new(id: &str, model: &str, created: u64) -> Self { + pub fn new(envelope: &Arc) -> Self { Self { - id: id.to_string(), - object: "text_completion".to_string(), - created, - model: model.to_string(), + envelope: Arc::clone(envelope), choices: Vec::new(), usage: None, } diff --git a/rust/src/server/src/routes/openai/utils/types.rs b/rust/src/server/src/routes/openai/utils/types.rs index b079bf4857..0ed56a6e6c 100644 --- a/rust/src/server/src/routes/openai/utils/types.rs +++ b/rust/src/server/src/routes/openai/utils/types.rs @@ -201,6 +201,29 @@ pub struct InputAudio { // Streaming // ============================================================================ +#[derive(Debug, Serialize)] +pub(crate) struct StreamResponseEnvelope { + id: String, + object: &'static str, + created: u64, + model: String, +} + +impl StreamResponseEnvelope { + pub(crate) fn new(id: String, object: &'static str, created: u64, model: String) -> Self { + Self { + id, + object, + created, + model, + } + } + + pub(crate) fn model(&self) -> &str { + &self.model + } +} + /// Mirrors the Python vLLM `StreamOptions` class. #[derive(Debug, Clone, Deserialize, Serialize)] pub struct StreamOptions { diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 95bae1939e..0912f78ab4 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -170,6 +170,22 @@ fn sse_json_payloads(text: &str) -> Vec { .collect() } +fn assert_shared_openai_stream_envelope(payloads: &[serde_json::Value], object: &str, model: &str) { + let first = payloads.first().expect("at least one SSE JSON payload"); + assert!(first["id"].as_str().is_some_and(|id| !id.is_empty())); + assert!(first["created"].is_u64()); + assert_eq!(first["object"], object); + assert_eq!(first["model"], model); + + for payload in payloads { + assert_eq!(payload["id"], first["id"]); + assert_eq!(payload["object"], first["object"]); + assert_eq!(payload["created"], first["created"]); + assert_eq!(payload["model"], first["model"]); + assert!(payload.get("envelope").is_none()); + } +} + type TestFuture<'a> = Pin + Send + 'a>>; fn boxed_test_future<'a>(future: impl Future + Send + 'a) -> TestFuture<'a> { @@ -2717,6 +2733,11 @@ async fn happy_path_returns_sse_stream() { let text = String::from_utf8(body.to_vec()).expect("utf8 body"); let after = METRICS.render().unwrap(); + assert_shared_openai_stream_envelope( + &sse_json_payloads(&text), + "chat.completion.chunk", + "Qwen/Qwen1.5-0.5B-Chat", + ); assert!(text.contains("\"role\":\"assistant\""), "{text}"); assert!(text.starts_with("data: "), "{text}"); assert_eq!( @@ -4465,6 +4486,11 @@ async fn completions_happy_path_returns_sse_stream() { engine_task.await.expect("mock engine task"); let text = String::from_utf8(body.to_vec()).expect("utf8 body"); let payloads = sse_data_payloads(&text); + assert_shared_openai_stream_envelope( + &sse_json_payloads(&text), + "text_completion", + "Qwen/Qwen1.5-0.5B-Chat", + ); let usage_index = payloads .iter() .position(|payload| payload.contains("\"usage\":"))