[Rust Frontend] Optimize SSE streaming hot path (#51321)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Signed-off-by: reidliu41 <reid201711@gmail.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Reid
2026-08-28 20:35:38 +00:00
committed by GitHub
co-authored by Bugen Zhao OpenAI Codex
parent 06cccf8730
commit 74850f9f66
8 changed files with 204 additions and 196 deletions
@@ -395,16 +395,18 @@ async fn generate_sse_stream(
}
fn to_sse_event(chunk: &GenerateStreamResponse) -> Event {
let payload = serde_json::to_string(chunk).expect("generate chunk must serialize to JSON");
trace!(payload, "generate emitting chunk");
Event::default().data(payload)
trace!(?chunk, "generate emitting chunk");
Event::default()
.json_data(chunk)
.expect("generate chunk must serialize to JSON")
}
fn to_error_sse_event(error: &ApiError) -> Event {
let payload = serde_json::to_string(&error.to_error_response())
.expect("ErrorResponse must serialize to JSON");
trace!(payload, "generate emitting error");
Event::default().data(payload)
let response = error.to_error_response();
trace!(?response, "generate emitting error");
Event::default()
.json_data(response)
.expect("ErrorResponse must serialize to JSON")
}
fn done_sse_event() -> Event {
@@ -39,7 +39,8 @@ use crate::routes::openai::utils::logprobs::{
decoded_logprobs_to_openai_chat, prompt_logprobs_to_maps,
};
use crate::routes::openai::utils::types::{
ChatLogProbs, FunctionCallDelta, FunctionCallResponse, ToolCall, ToolCallDelta, Usage,
ChatLogProbs, FunctionCallDelta, FunctionCallResponse, StreamResponseEnvelope, ToolCall,
ToolCallDelta, Usage,
};
use crate::routes::openai::utils::usage::ContinuousUsage;
use crate::routes::openai::utils::validated_json::ValidatedJson;
@@ -270,6 +271,12 @@ async fn chat_completion_chunk_stream(
}: ResponseOptions,
mut y: TryYielder<ChatCompletionStreamResponse, ApiError>,
) -> Result<(), ApiError> {
let envelope = Arc::new(StreamResponseEnvelope::new(
request_id,
"chat.completion.chunk",
created,
response_model,
));
let mut saw_tool_calls = false;
// `LogprobsDelta` is emitted after all chat events for one decoded update.
// If that update contains hidden reasoning, including delimiter-only block
@@ -303,7 +310,7 @@ async fn chat_completion_chunk_stream(
prompt_token_ids, ..
}) => {
continuous_usage.set_prompt_tokens(prompt_token_ids.len());
let mut chunk = start_chunk(&request_id, &response_model, created);
let mut chunk = start_chunk(&envelope);
if return_token_ids {
chunk.prompt_token_ids = Some(prompt_token_ids.to_vec());
}
@@ -311,9 +318,7 @@ async fn chat_completion_chunk_stream(
// When echo=true, emit the last assistant message content as a delta chunk.
if let Some(echo_text) = &echo {
yield_chunk!(block_delta_chunk(
&request_id,
&response_model,
created,
&envelope,
AssistantBlockKind::Text,
echo_text.clone(),
));
@@ -326,13 +331,7 @@ async fn chat_completion_chunk_stream(
if let Some(pending_chunk) = pending_chunk.as_mut() {
pending_chunk.push_block_delta(kind, delta);
} else {
yield_chunk!(block_delta_chunk(
&request_id,
&response_model,
created,
kind,
delta,
));
yield_chunk!(block_delta_chunk(&envelope, kind, delta));
}
} else {
suppress_current_update_metadata = true;
@@ -368,18 +367,11 @@ async fn chat_completion_chunk_stream(
if let Some(pending_chunk) = pending_chunk.as_mut() {
pending_chunk.logprobs = openai_logprobs;
pending_chunk.token_ids = openai_token_ids;
if let Some(chunk) =
pending_chunk.take_chunk(&request_id, &response_model, created)
{
if let Some(chunk) = pending_chunk.take_chunk(&envelope) {
yield_chunk!(chunk);
}
} else if let Some(logprobs) = openai_logprobs {
yield_chunk!(logprobs_only_chunk(
&request_id,
&response_model,
created,
logprobs,
));
yield_chunk!(logprobs_only_chunk(&envelope, logprobs));
}
}
Ok(ChatEvent::BlockStart { kind, .. }) => {
@@ -407,14 +399,7 @@ async fn chat_completion_chunk_stream(
if let Some(pending_chunk) = pending_chunk.as_mut() {
pending_chunk.push_tool_call_start(tool_index, id, name);
} else {
yield_chunk!(tool_call_start_chunk(
&request_id,
&response_model,
created,
tool_index,
id,
name,
));
yield_chunk!(tool_call_start_chunk(&envelope, tool_index, id, name));
}
}
Ok(ChatEvent::ToolCallArgumentsDelta { index, delta }) => {
@@ -422,13 +407,7 @@ async fn chat_completion_chunk_stream(
if let Some(pending_chunk) = pending_chunk.as_mut() {
pending_chunk.push_tool_call_arguments(tool_index, delta);
} else {
yield_chunk!(tool_call_arguments_chunk(
&request_id,
&response_model,
created,
tool_index,
delta,
));
yield_chunk!(tool_call_arguments_chunk(&envelope, tool_index, delta));
}
}
Ok(ChatEvent::ToolCallEnd { .. }) => {
@@ -442,7 +421,7 @@ async fn chat_completion_chunk_stream(
if enable_log_requests {
info!(
stream = true,
model = %response_model,
model = %envelope.model(),
prompt_tokens = final_usage.prompt_token_count,
output_tokens = final_usage.output_token_count,
finish_reason = finish_reason.as_str(),
@@ -456,16 +435,13 @@ async fn chat_completion_chunk_stream(
);
if let Some(pending_chunk) = pending_chunk.as_mut()
&& let Some(chunk) =
pending_chunk.take_chunk(&request_id, &response_model, created)
&& let Some(chunk) = pending_chunk.take_chunk(&envelope)
{
yield_chunk!(chunk);
}
match final_chunk(
&request_id,
&response_model,
created,
&envelope,
finish_reason,
saw_tool_calls && !is_named_tool_choice,
) {
@@ -481,9 +457,7 @@ async fn chat_completion_chunk_stream(
if include_usage {
y.yield_ok(usage_chunk(
&request_id,
&response_model,
created,
&envelope,
Usage::from_token_usage(final_usage, enable_prompt_tokens_details),
))
.await;
@@ -504,12 +478,10 @@ async fn chat_completion_chunk_stream(
}
fn usage_chunk(
request_id: &str,
response_model: &str,
created: u64,
envelope: &Arc<StreamResponseEnvelope>,
usage: Usage,
) -> ChatCompletionStreamResponse {
let mut chunk = ChatCompletionStreamResponse::new(request_id, response_model, created);
let mut chunk = ChatCompletionStreamResponse::new(envelope);
chunk.usage = Some(usage);
chunk
}
@@ -588,9 +560,7 @@ impl PendingChatChunk {
/// necessarily with a visible/chat-semantic delta.
fn take_chunk(
&mut self,
request_id: &str,
response_model: &str,
created: u64,
envelope: &Arc<StreamResponseEnvelope>,
) -> Option<ChatCompletionStreamResponse> {
let has_delta = self.delta.content.is_some()
|| self.delta.reasoning.is_some()
@@ -601,7 +571,7 @@ impl PendingChatChunk {
return None;
}
let mut chunk = ChatCompletionStreamResponse::new(request_id, response_model, created);
let mut chunk = ChatCompletionStreamResponse::new(envelope);
chunk.choices.push(ChatCompletionStreamChoice {
delta: self.take_delta(),
logprobs,
@@ -659,18 +629,19 @@ async fn chat_completion_sse_stream(
/// Serialize one OpenAI chunk payload into one SSE `data:` event.
fn to_sse_event(chunk: &ChatCompletionStreamResponse) -> Event {
let payload =
serde_json::to_string(chunk).expect("ChatCompletionStreamResponse must serialize to JSON");
trace!(payload, "chat completion emitting chunk");
Event::default().data(payload)
trace!(?chunk, "chat completion emitting chunk");
Event::default()
.json_data(chunk)
.expect("ChatCompletionStreamResponse must serialize to JSON")
}
/// Serialize one OpenAI error payload into one SSE `data:` event.
fn to_error_sse_event(error: &ApiError) -> Event {
let payload = serde_json::to_string(&error.to_error_response())
.expect("ErrorResponse must serialize to JSON");
trace!(payload, "chat completion emitting error");
Event::default().data(payload)
let response = error.to_error_response();
trace!(?response, "chat completion emitting error");
Event::default()
.json_data(response)
.expect("ErrorResponse must serialize to JSON")
}
/// Build the terminal OpenAI SSE sentinel event.
@@ -681,12 +652,8 @@ fn done_sse_event() -> Event {
/// Build the initial assistant-role SSE chunk required by the OpenAI streaming
/// protocol.
fn start_chunk(
request_id: &str,
response_model: &str,
created: u64,
) -> ChatCompletionStreamResponse {
let mut chunk = ChatCompletionStreamResponse::new(request_id, response_model, created);
fn start_chunk(envelope: &Arc<StreamResponseEnvelope>) -> ChatCompletionStreamResponse {
let mut chunk = ChatCompletionStreamResponse::new(envelope);
chunk.choices.push(ChatCompletionStreamChoice {
delta: ChatMessageDelta {
role: Some(AssistantRole),
@@ -699,9 +666,7 @@ fn start_chunk(
/// Build one content-delta SSE chunk from one internal assistant block delta.
fn block_delta_chunk(
request_id: &str,
response_model: &str,
created: u64,
envelope: &Arc<StreamResponseEnvelope>,
kind: AssistantBlockKind,
delta: String,
) -> ChatCompletionStreamResponse {
@@ -719,7 +684,7 @@ fn block_delta_chunk(
}
};
let mut chunk = ChatCompletionStreamResponse::new(request_id, response_model, created);
let mut chunk = ChatCompletionStreamResponse::new(envelope);
chunk.choices.push(ChatCompletionStreamChoice {
delta,
..Default::default()
@@ -728,14 +693,12 @@ fn block_delta_chunk(
}
fn tool_call_start_chunk(
request_id: &str,
response_model: &str,
created: u64,
envelope: &Arc<StreamResponseEnvelope>,
tool_index: u32,
id: String,
name: String,
) -> ChatCompletionStreamResponse {
let mut chunk = ChatCompletionStreamResponse::new(request_id, response_model, created);
let mut chunk = ChatCompletionStreamResponse::new(envelope);
chunk.choices.push(ChatCompletionStreamChoice {
delta: ChatMessageDelta {
tool_calls: Some(vec![ToolCallDelta {
@@ -755,13 +718,11 @@ fn tool_call_start_chunk(
}
fn tool_call_arguments_chunk(
request_id: &str,
response_model: &str,
created: u64,
envelope: &Arc<StreamResponseEnvelope>,
tool_index: u32,
delta: String,
) -> ChatCompletionStreamResponse {
let mut chunk = ChatCompletionStreamResponse::new(request_id, response_model, created);
let mut chunk = ChatCompletionStreamResponse::new(envelope);
chunk.choices.push(ChatCompletionStreamChoice {
delta: ChatMessageDelta {
tool_calls: Some(vec![ToolCallDelta {
@@ -781,12 +742,10 @@ fn tool_call_arguments_chunk(
}
fn logprobs_only_chunk(
request_id: &str,
response_model: &str,
created: u64,
envelope: &Arc<StreamResponseEnvelope>,
logprobs: ChatLogProbs,
) -> ChatCompletionStreamResponse {
let mut chunk = ChatCompletionStreamResponse::new(request_id, response_model, created);
let mut chunk = ChatCompletionStreamResponse::new(envelope);
chunk.choices.push(ChatCompletionStreamChoice {
logprobs: Some(logprobs),
..Default::default()
@@ -796,9 +755,7 @@ fn logprobs_only_chunk(
/// Build the terminal SSE chunk carrying the OpenAI finish reason.
fn final_chunk(
request_id: &str,
response_model: &str,
created: u64,
envelope: &Arc<StreamResponseEnvelope>,
finish_reason: FinishReason,
use_tool_calls_finish_reason: bool,
) -> Result<ChatCompletionStreamResponse, ApiError> {
@@ -811,7 +768,7 @@ fn final_chunk(
"chat stream finished"
);
let mut chunk = ChatCompletionStreamResponse::new(request_id, response_model, created);
let mut chunk = ChatCompletionStreamResponse::new(envelope);
chunk.choices.push(ChatCompletionStreamChoice {
finish_reason: Some(finish_reason.to_string()),
stop_reason,
@@ -844,6 +801,8 @@ fn stop_reason_to_json(stop_reason: &StopReason) -> Value {
#[cfg(test)]
mod tests {
use std::sync::Arc;
use futures::{StreamExt as _, stream};
use serde_json::json;
use vllm_chat::{
@@ -853,19 +812,23 @@ mod tests {
use vllm_text::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTokenLogprob};
use super::{
ApiServerOptions, ResponseOptions, block_delta_chunk, chat_completion_chunk_stream,
final_chunk,
ApiServerOptions, ResponseOptions, StreamResponseEnvelope, block_delta_chunk,
chat_completion_chunk_stream, final_chunk,
};
fn stream_envelope() -> Arc<StreamResponseEnvelope> {
Arc::new(StreamResponseEnvelope::new(
"chatcmpl-1".to_string(),
"chat.completion.chunk",
1,
"model".to_string(),
))
}
#[test]
fn text_chunk_uses_content_only_delta() {
let chunk = block_delta_chunk(
"chatcmpl-1",
"model",
1,
AssistantBlockKind::Text,
"hello".to_string(),
);
let envelope = stream_envelope();
let chunk = block_delta_chunk(&envelope, AssistantBlockKind::Text, "hello".to_string());
assert_eq!(chunk.choices[0].delta.role, None);
assert_eq!(chunk.choices[0].delta.content.as_deref(), Some("hello"));
assert_eq!(chunk.choices[0].delta.reasoning, None);
@@ -873,10 +836,9 @@ mod tests {
#[test]
fn reasoning_chunk_uses_reasoning_only_delta() {
let envelope = stream_envelope();
let chunk = block_delta_chunk(
"chatcmpl-1",
"model",
1,
&envelope,
AssistantBlockKind::Reasoning,
"thinking".to_string(),
);
@@ -890,10 +852,9 @@ mod tests {
#[test]
fn final_chunk_maps_stop_finish_reason_and_stop_reason() {
let envelope = stream_envelope();
let chunk = final_chunk(
"chatcmpl-1",
"model",
1,
&envelope,
FinishReason::Stop(Some(StopReason::Text("stop".to_string()))),
false,
)
@@ -905,7 +866,7 @@ mod tests {
#[test]
fn final_chunk_maps_length_finish_reason() {
let chunk = final_chunk("chatcmpl-1", "model", 1, FinishReason::Length, false)
let chunk = final_chunk(&stream_envelope(), FinishReason::Length, false)
.expect("finish reason is valid");
assert_eq!(chunk.choices[0].finish_reason.as_deref(), Some("length"));
@@ -914,7 +875,7 @@ mod tests {
#[test]
fn final_chunk_maps_abort_finish_reason() {
let chunk = final_chunk("chatcmpl-1", "model", 1, FinishReason::Abort, false)
let chunk = final_chunk(&stream_envelope(), FinishReason::Abort, false)
.expect("abort is a valid finish reason");
assert_eq!(chunk.choices[0].finish_reason.as_deref(), Some("abort"));
@@ -923,12 +884,12 @@ mod tests {
#[test]
fn final_chunk_rejects_error_finish_reason() {
assert!(final_chunk("chatcmpl-1", "model", 1, FinishReason::Error, false).is_err());
assert!(final_chunk(&stream_envelope(), FinishReason::Error, false).is_err());
}
#[test]
fn final_chunk_maps_stop_to_tool_calls_when_tool_calls_were_streamed() {
let chunk = final_chunk("chatcmpl-1", "model", 1, FinishReason::stop_eos(), true)
let chunk = final_chunk(&stream_envelope(), FinishReason::stop_eos(), true)
.expect("finish reason is valid");
assert_eq!(
@@ -1002,6 +963,7 @@ mod tests {
.expect("stream chunks");
assert_eq!(chunks.len(), 4);
assert!(chunks.windows(2).all(|pair| Arc::ptr_eq(&pair[0].envelope, &pair[1].envelope)));
assert_eq!(chunks[1].choices[0].delta.content.as_deref(), Some("hi"));
let logprobs = chunks[1].choices[0].logprobs.as_ref().expect("logprobs");
let content = logprobs.content.as_ref().expect("logprobs content");
@@ -3,6 +3,7 @@
use std::collections::HashMap;
use std::fmt;
use std::sync::Arc;
use serde::{Deserialize, Serialize};
use serde_json::Value;
@@ -13,9 +14,9 @@ use vllm_engine_core_client::protocol::sampling::RepetitionDetectionParams;
use crate::routes::openai::utils::structured_outputs::ResponseFormat;
use crate::routes::openai::utils::types::{
ChatLogProbs, ChatMessage, Normalizable, PromptLogprobs, StreamOptions, StringOrArray, Tool,
ToolCall, ToolCallDelta, ToolChoice, Usage, default_true, deserialize_request_top_k,
validate_messages, validate_stop, validate_top_p_value,
ChatLogProbs, ChatMessage, Normalizable, PromptLogprobs, StreamOptions, StreamResponseEnvelope,
StringOrArray, Tool, ToolCall, ToolCallDelta, ToolChoice, Usage, default_true,
deserialize_request_top_k, validate_messages, validate_stop, validate_top_p_value,
};
/// vLLM-compatible request type for the Chat Completions API.
@@ -383,10 +384,8 @@ pub(super) struct ChatCompletionMessage {
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Serialize)]
pub(super) struct ChatCompletionStreamResponse {
pub id: String,
pub object: String,
pub created: u64,
pub model: String,
#[serde(flatten)]
pub envelope: Arc<StreamResponseEnvelope>,
pub choices: Vec<ChatCompletionStreamChoice>,
pub usage: Option<Usage>,
pub prompt_token_ids: Option<Vec<u32>>,
@@ -394,12 +393,9 @@ pub(super) struct ChatCompletionStreamResponse {
impl ChatCompletionStreamResponse {
/// Create a stream response with the standard envelope fields pre-filled.
pub fn new(id: &str, model: &str, created: u64) -> Self {
pub fn new(envelope: &Arc<StreamResponseEnvelope>) -> Self {
Self {
id: id.to_string(),
object: "chat.completion.chunk".to_string(),
created,
model: model.to_string(),
envelope: Arc::clone(envelope),
choices: Vec::new(),
usage: None,
prompt_token_ids: None,
@@ -33,7 +33,7 @@ use super::utils::logprobs::{
collected_logprobs_to_openai, decoded_logprobs_to_openai, decoded_prompt_logprobs_to_openai,
prompt_logprobs_to_maps, text_len,
};
use super::utils::types::Usage;
use super::utils::types::{StreamResponseEnvelope, Usage};
use crate::config::ApiServerOptions;
use crate::error::{ApiError, bail_server_error, server_error, text_submit_error};
use crate::lora::LoraModelResolution;
@@ -257,6 +257,12 @@ async fn completion_chunk_stream(
mut y: TryYielder<CompletionSseChunk, ApiError>,
) -> Result<(), ApiError> {
pin_mut!(stream);
let envelope = Arc::new(StreamResponseEnvelope::new(
request_id,
"text_completion",
created,
response_model,
));
let mut visible_text_len = 0_u32;
let mut first_chunk = true;
let mut continuous_usage = ContinuousUsage::default();
@@ -292,13 +298,7 @@ async fn completion_chunk_stream(
} else {
None
};
let mut chunk = delta_chunk(
&request_id,
&response_model,
created,
prompt.clone(),
logprobs,
);
let mut chunk = delta_chunk(&envelope, prompt.clone(), logprobs);
if return_token_ids && first_chunk {
if let Some(choice) = chunk.choices.first_mut() {
choice.prompt_token_ids = Some(prompt_token_ids.to_vec());
@@ -308,8 +308,7 @@ async fn completion_chunk_stream(
yield_chunk!(chunk);
} else if return_token_ids {
// Emit a chunk with prompt_token_ids in the first streaming response
let mut chunk =
delta_chunk(&request_id, &response_model, created, String::new(), None);
let mut chunk = delta_chunk(&envelope, String::new(), None);
if let Some(choice) = chunk.choices.first_mut() {
choice.prompt_token_ids = Some(prompt_token_ids.to_vec());
}
@@ -331,7 +330,7 @@ async fn completion_chunk_stream(
if enable_log_requests {
info!(
stream = true,
model = %response_model,
model = %envelope.model(),
prompt_tokens = finished.usage.prompt_token_count,
output_tokens = finished.usage.output_token_count,
finish_reason = finished.finish_reason.as_str(),
@@ -342,19 +341,12 @@ async fn completion_chunk_stream(
finished.usage.prompt_token_count,
finished.usage.output_token_count,
);
let final_chunk = final_chunk(
&request_id,
&response_model,
created,
finished.finish_reason,
)?;
let final_chunk = final_chunk(&envelope, finished.finish_reason)?;
yield_chunk!(final_chunk);
if include_usage {
y.yield_ok(CompletionSseChunk::Usage(usage_chunk(
&request_id,
&response_model,
created,
&envelope,
Usage::from_token_usage(
finished.usage,
enable_prompt_tokens_details,
@@ -380,7 +372,7 @@ async fn completion_chunk_stream(
} else {
None
};
let mut chunk = delta_chunk(&request_id, &response_model, created, delta, logprobs);
let mut chunk = delta_chunk(&envelope, delta, logprobs);
let delta_token_count = token_ids.len();
continuous_usage.add_output_tokens(delta_token_count);
if return_token_ids && let Some(choice) = chunk.choices.first_mut() {
@@ -393,7 +385,7 @@ async fn completion_chunk_stream(
if enable_log_requests {
info!(
stream = true,
model = %response_model,
model = %envelope.model(),
prompt_tokens = finished.usage.prompt_token_count,
output_tokens = finished.usage.output_token_count,
finish_reason = finished.finish_reason.as_str(),
@@ -404,19 +396,12 @@ async fn completion_chunk_stream(
finished.usage.prompt_token_count,
finished.usage.output_token_count,
);
let final_chunk = final_chunk(
&request_id,
&response_model,
created,
finished.finish_reason,
)?;
let final_chunk = final_chunk(&envelope, finished.finish_reason)?;
yield_chunk!(final_chunk);
if include_usage {
y.yield_ok(CompletionSseChunk::Usage(usage_chunk(
&request_id,
&response_model,
created,
&envelope,
Usage::from_token_usage(finished.usage, enable_prompt_tokens_details),
)))
.await;
@@ -436,13 +421,11 @@ async fn completion_chunk_stream(
}
fn delta_chunk(
request_id: &str,
response_model: &str,
created: u64,
envelope: &Arc<StreamResponseEnvelope>,
text: String,
logprobs: Option<LogProbs>,
) -> CompletionStreamResponse {
let mut chunk = CompletionStreamResponse::new(request_id, response_model, created);
let mut chunk = CompletionStreamResponse::new(envelope);
chunk.choices.push(CompletionStreamChoice {
text,
logprobs,
@@ -452,15 +435,13 @@ fn delta_chunk(
}
fn final_chunk(
request_id: &str,
response_model: &str,
created: u64,
envelope: &Arc<StreamResponseEnvelope>,
finish_reason: FinishReason,
) -> Result<CompletionStreamResponse, ApiError> {
let stop_reason = finish_reason.as_stop_reason().map(stop_reason_to_json);
let finish_reason = completion_finish_reason_to_openai(&finish_reason)?;
let mut chunk = CompletionStreamResponse::new(request_id, response_model, created);
let mut chunk = CompletionStreamResponse::new(envelope);
chunk.choices.push(CompletionStreamChoice {
finish_reason: Some(finish_reason.to_string()),
stop_reason,
@@ -517,13 +498,8 @@ fn prompt_only_logprobs_to_openai(
))
}
fn usage_chunk(
request_id: &str,
response_model: &str,
created: u64,
usage: Usage,
) -> CompletionStreamResponse {
let mut chunk = CompletionStreamResponse::new(request_id, response_model, created);
fn usage_chunk(envelope: &Arc<StreamResponseEnvelope>, usage: Usage) -> CompletionStreamResponse {
let mut chunk = CompletionStreamResponse::new(envelope);
chunk.usage = Some(usage);
chunk
}
@@ -556,17 +532,19 @@ async fn completion_sse_stream(
/// Serialize one OpenAI chunk payload into one SSE `data:` event.
fn to_sse_event(chunk: &CompletionSseChunk) -> Event {
let payload = serde_json::to_string(chunk).expect("completion chunk must serialize to JSON");
trace!(payload, "completion emitting chunk");
Event::default().data(payload)
trace!(?chunk, "completion emitting chunk");
Event::default()
.json_data(chunk)
.expect("completion chunk must serialize to JSON")
}
/// Serialize one OpenAI error payload into one SSE `data:` event.
fn to_error_sse_event(error: &ApiError) -> Event {
let payload = serde_json::to_string(&error.to_error_response())
.expect("ErrorResponse must serialize to JSON");
trace!(payload, "completion emitting error");
Event::default().data(payload)
let response = error.to_error_response();
trace!(?response, "completion emitting error");
Event::default()
.json_data(response)
.expect("ErrorResponse must serialize to JSON")
}
/// Build the terminal OpenAI SSE sentinel event.
@@ -577,6 +555,8 @@ fn done_sse_event() -> Event {
#[cfg(test)]
mod tests {
use std::sync::Arc;
use futures::{StreamExt as _, stream};
use itertools::Itertools as _;
use vllm_engine_core_client::protocol::output::StopReason;
@@ -586,13 +566,29 @@ mod tests {
};
use super::{
ApiServerOptions, CompletionSseChunk, ResponseOptions, completion_chunk_stream, final_chunk,
ApiServerOptions, CompletionSseChunk, CompletionStreamResponse, ResponseOptions,
StreamResponseEnvelope, completion_chunk_stream, final_chunk,
};
fn stream_envelope() -> Arc<StreamResponseEnvelope> {
Arc::new(StreamResponseEnvelope::new(
"cmpl-1".to_string(),
"text_completion",
1,
"model".to_string(),
))
}
fn chunk_response(chunk: &CompletionSseChunk) -> &CompletionStreamResponse {
match chunk {
CompletionSseChunk::Chunk(response) | CompletionSseChunk::Usage(response) => response,
}
}
#[test]
fn final_chunk_maps_stop_finish_reason() {
let chunk = final_chunk("cmpl-1", "model", 1, FinishReason::stop_eos())
.expect("finish reason valid");
let chunk =
final_chunk(&stream_envelope(), FinishReason::stop_eos()).expect("finish reason valid");
assert_eq!(chunk.choices[0].finish_reason.as_deref(), Some("stop"));
assert_eq!(chunk.choices[0].text, "");
}
@@ -600,20 +596,20 @@ mod tests {
#[test]
fn final_chunk_maps_length_finish_reason() {
let chunk =
final_chunk("cmpl-1", "model", 1, FinishReason::Length).expect("finish reason valid");
final_chunk(&stream_envelope(), FinishReason::Length).expect("finish reason valid");
assert_eq!(chunk.choices[0].finish_reason.as_deref(), Some("length"));
}
#[test]
fn final_chunk_maps_abort_finish_reason() {
let chunk =
final_chunk("cmpl-1", "model", 1, FinishReason::Abort).expect("finish reason valid");
final_chunk(&stream_envelope(), FinishReason::Abort).expect("finish reason valid");
assert_eq!(chunk.choices[0].finish_reason.as_deref(), Some("abort"));
}
#[test]
fn final_chunk_rejects_error_finish_reason() {
assert!(final_chunk("cmpl-1", "model", 1, FinishReason::Error).is_err());
assert!(final_chunk(&stream_envelope(), FinishReason::Error).is_err());
}
#[tokio::test]
@@ -702,6 +698,13 @@ mod tests {
let chunks: Vec<_> = chunks.into_iter().try_collect().expect("stream should succeed");
assert!(chunks.windows(2).all(|pair| {
Arc::ptr_eq(
&chunk_response(&pair[0]).envelope,
&chunk_response(&pair[1]).envelope,
)
}));
match &chunks[0] {
CompletionSseChunk::Chunk(chunk) => {
assert_eq!(chunk.choices[0].text, "h");
@@ -2,6 +2,7 @@
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::collections::HashMap;
use std::sync::Arc;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
@@ -10,8 +11,8 @@ use vllm_engine_core_client::protocol::sampling::RepetitionDetectionParams;
use vllm_text::Prompt;
use crate::routes::openai::utils::types::{
LogProbs, Normalizable, PromptLogprobs, StreamOptions, StringOrArray, Usage, default_true,
deserialize_request_top_k, validate_stop,
LogProbs, Normalizable, PromptLogprobs, StreamOptions, StreamResponseEnvelope, StringOrArray,
Usage, default_true, deserialize_request_top_k, validate_stop,
};
/// Serde default for `CompletionRequest::max_tokens`, matching the Python vLLM
@@ -241,22 +242,17 @@ pub(super) struct CompletionChoice {
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Serialize)]
pub(super) struct CompletionStreamResponse {
pub id: String,
pub object: String,
pub created: u64,
pub model: String,
#[serde(flatten)]
pub envelope: Arc<StreamResponseEnvelope>,
pub choices: Vec<CompletionStreamChoice>,
pub usage: Option<Usage>,
}
impl CompletionStreamResponse {
/// Create a stream response with the standard envelope fields pre-filled.
pub fn new(id: &str, model: &str, created: u64) -> Self {
pub fn new(envelope: &Arc<StreamResponseEnvelope>) -> Self {
Self {
id: id.to_string(),
object: "text_completion".to_string(),
created,
model: model.to_string(),
envelope: Arc::clone(envelope),
choices: Vec::new(),
usage: None,
}
@@ -201,6 +201,29 @@ pub struct InputAudio {
// Streaming
// ============================================================================
#[derive(Debug, Serialize)]
pub(crate) struct StreamResponseEnvelope {
id: String,
object: &'static str,
created: u64,
model: String,
}
impl StreamResponseEnvelope {
pub(crate) fn new(id: String, object: &'static str, created: u64, model: String) -> Self {
Self {
id,
object,
created,
model,
}
}
pub(crate) fn model(&self) -> &str {
&self.model
}
}
/// Mirrors the Python vLLM `StreamOptions` class.
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct StreamOptions {
+26
View File
@@ -170,6 +170,22 @@ fn sse_json_payloads(text: &str) -> Vec<serde_json::Value> {
.collect()
}
fn assert_shared_openai_stream_envelope(payloads: &[serde_json::Value], object: &str, model: &str) {
let first = payloads.first().expect("at least one SSE JSON payload");
assert!(first["id"].as_str().is_some_and(|id| !id.is_empty()));
assert!(first["created"].is_u64());
assert_eq!(first["object"], object);
assert_eq!(first["model"], model);
for payload in payloads {
assert_eq!(payload["id"], first["id"]);
assert_eq!(payload["object"], first["object"]);
assert_eq!(payload["created"], first["created"]);
assert_eq!(payload["model"], first["model"]);
assert!(payload.get("envelope").is_none());
}
}
type TestFuture<'a> = Pin<Box<dyn Future<Output = ()> + Send + 'a>>;
fn boxed_test_future<'a>(future: impl Future<Output = ()> + Send + 'a) -> TestFuture<'a> {
@@ -2717,6 +2733,11 @@ async fn happy_path_returns_sse_stream() {
let text = String::from_utf8(body.to_vec()).expect("utf8 body");
let after = METRICS.render().unwrap();
assert_shared_openai_stream_envelope(
&sse_json_payloads(&text),
"chat.completion.chunk",
"Qwen/Qwen1.5-0.5B-Chat",
);
assert!(text.contains("\"role\":\"assistant\""), "{text}");
assert!(text.starts_with("data: "), "{text}");
assert_eq!(
@@ -4465,6 +4486,11 @@ async fn completions_happy_path_returns_sse_stream() {
engine_task.await.expect("mock engine task");
let text = String::from_utf8(body.to_vec()).expect("utf8 body");
let payloads = sse_data_payloads(&text);
assert_shared_openai_stream_envelope(
&sse_json_payloads(&text),
"text_completion",
"Qwen/Qwen1.5-0.5B-Chat",
);
let usage_index = payloads
.iter()
.position(|payload| payload.contains("\"usage\":"))