[Rust Frontend] Optimize SSE streaming hot path (#51321)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: OpenAI Codex <codex@openai.com> Signed-off-by: reidliu41 <reid201711@gmail.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Bugen Zhao
OpenAI Codex
parent
06cccf8730
commit
74850f9f66
+1
-1
@@ -90,7 +90,7 @@ rlimit = "0.11.0"
|
||||
rmp-serde = "1.3.1"
|
||||
rmpv = { version = "1.3.1", features = ["with-serde"] }
|
||||
rustc-hash = "1.1.0"
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde = { version = "1.0.228", features = ["derive", "rc"] }
|
||||
serde-json-fmt = "0.1.0"
|
||||
serde_default = "0.2.0"
|
||||
serde_json = { version = "1.0.145", features = ["preserve_order"] }
|
||||
|
||||
@@ -395,16 +395,18 @@ async fn generate_sse_stream(
|
||||
}
|
||||
|
||||
fn to_sse_event(chunk: &GenerateStreamResponse) -> Event {
|
||||
let payload = serde_json::to_string(chunk).expect("generate chunk must serialize to JSON");
|
||||
trace!(payload, "generate emitting chunk");
|
||||
Event::default().data(payload)
|
||||
trace!(?chunk, "generate emitting chunk");
|
||||
Event::default()
|
||||
.json_data(chunk)
|
||||
.expect("generate chunk must serialize to JSON")
|
||||
}
|
||||
|
||||
fn to_error_sse_event(error: &ApiError) -> Event {
|
||||
let payload = serde_json::to_string(&error.to_error_response())
|
||||
.expect("ErrorResponse must serialize to JSON");
|
||||
trace!(payload, "generate emitting error");
|
||||
Event::default().data(payload)
|
||||
let response = error.to_error_response();
|
||||
trace!(?response, "generate emitting error");
|
||||
Event::default()
|
||||
.json_data(response)
|
||||
.expect("ErrorResponse must serialize to JSON")
|
||||
}
|
||||
|
||||
fn done_sse_event() -> Event {
|
||||
|
||||
@@ -39,7 +39,8 @@ use crate::routes::openai::utils::logprobs::{
|
||||
decoded_logprobs_to_openai_chat, prompt_logprobs_to_maps,
|
||||
};
|
||||
use crate::routes::openai::utils::types::{
|
||||
ChatLogProbs, FunctionCallDelta, FunctionCallResponse, ToolCall, ToolCallDelta, Usage,
|
||||
ChatLogProbs, FunctionCallDelta, FunctionCallResponse, StreamResponseEnvelope, ToolCall,
|
||||
ToolCallDelta, Usage,
|
||||
};
|
||||
use crate::routes::openai::utils::usage::ContinuousUsage;
|
||||
use crate::routes::openai::utils::validated_json::ValidatedJson;
|
||||
@@ -270,6 +271,12 @@ async fn chat_completion_chunk_stream(
|
||||
}: ResponseOptions,
|
||||
mut y: TryYielder<ChatCompletionStreamResponse, ApiError>,
|
||||
) -> Result<(), ApiError> {
|
||||
let envelope = Arc::new(StreamResponseEnvelope::new(
|
||||
request_id,
|
||||
"chat.completion.chunk",
|
||||
created,
|
||||
response_model,
|
||||
));
|
||||
let mut saw_tool_calls = false;
|
||||
// `LogprobsDelta` is emitted after all chat events for one decoded update.
|
||||
// If that update contains hidden reasoning, including delimiter-only block
|
||||
@@ -303,7 +310,7 @@ async fn chat_completion_chunk_stream(
|
||||
prompt_token_ids, ..
|
||||
}) => {
|
||||
continuous_usage.set_prompt_tokens(prompt_token_ids.len());
|
||||
let mut chunk = start_chunk(&request_id, &response_model, created);
|
||||
let mut chunk = start_chunk(&envelope);
|
||||
if return_token_ids {
|
||||
chunk.prompt_token_ids = Some(prompt_token_ids.to_vec());
|
||||
}
|
||||
@@ -311,9 +318,7 @@ async fn chat_completion_chunk_stream(
|
||||
// When echo=true, emit the last assistant message content as a delta chunk.
|
||||
if let Some(echo_text) = &echo {
|
||||
yield_chunk!(block_delta_chunk(
|
||||
&request_id,
|
||||
&response_model,
|
||||
created,
|
||||
&envelope,
|
||||
AssistantBlockKind::Text,
|
||||
echo_text.clone(),
|
||||
));
|
||||
@@ -326,13 +331,7 @@ async fn chat_completion_chunk_stream(
|
||||
if let Some(pending_chunk) = pending_chunk.as_mut() {
|
||||
pending_chunk.push_block_delta(kind, delta);
|
||||
} else {
|
||||
yield_chunk!(block_delta_chunk(
|
||||
&request_id,
|
||||
&response_model,
|
||||
created,
|
||||
kind,
|
||||
delta,
|
||||
));
|
||||
yield_chunk!(block_delta_chunk(&envelope, kind, delta));
|
||||
}
|
||||
} else {
|
||||
suppress_current_update_metadata = true;
|
||||
@@ -368,18 +367,11 @@ async fn chat_completion_chunk_stream(
|
||||
if let Some(pending_chunk) = pending_chunk.as_mut() {
|
||||
pending_chunk.logprobs = openai_logprobs;
|
||||
pending_chunk.token_ids = openai_token_ids;
|
||||
if let Some(chunk) =
|
||||
pending_chunk.take_chunk(&request_id, &response_model, created)
|
||||
{
|
||||
if let Some(chunk) = pending_chunk.take_chunk(&envelope) {
|
||||
yield_chunk!(chunk);
|
||||
}
|
||||
} else if let Some(logprobs) = openai_logprobs {
|
||||
yield_chunk!(logprobs_only_chunk(
|
||||
&request_id,
|
||||
&response_model,
|
||||
created,
|
||||
logprobs,
|
||||
));
|
||||
yield_chunk!(logprobs_only_chunk(&envelope, logprobs));
|
||||
}
|
||||
}
|
||||
Ok(ChatEvent::BlockStart { kind, .. }) => {
|
||||
@@ -407,14 +399,7 @@ async fn chat_completion_chunk_stream(
|
||||
if let Some(pending_chunk) = pending_chunk.as_mut() {
|
||||
pending_chunk.push_tool_call_start(tool_index, id, name);
|
||||
} else {
|
||||
yield_chunk!(tool_call_start_chunk(
|
||||
&request_id,
|
||||
&response_model,
|
||||
created,
|
||||
tool_index,
|
||||
id,
|
||||
name,
|
||||
));
|
||||
yield_chunk!(tool_call_start_chunk(&envelope, tool_index, id, name));
|
||||
}
|
||||
}
|
||||
Ok(ChatEvent::ToolCallArgumentsDelta { index, delta }) => {
|
||||
@@ -422,13 +407,7 @@ async fn chat_completion_chunk_stream(
|
||||
if let Some(pending_chunk) = pending_chunk.as_mut() {
|
||||
pending_chunk.push_tool_call_arguments(tool_index, delta);
|
||||
} else {
|
||||
yield_chunk!(tool_call_arguments_chunk(
|
||||
&request_id,
|
||||
&response_model,
|
||||
created,
|
||||
tool_index,
|
||||
delta,
|
||||
));
|
||||
yield_chunk!(tool_call_arguments_chunk(&envelope, tool_index, delta));
|
||||
}
|
||||
}
|
||||
Ok(ChatEvent::ToolCallEnd { .. }) => {
|
||||
@@ -442,7 +421,7 @@ async fn chat_completion_chunk_stream(
|
||||
if enable_log_requests {
|
||||
info!(
|
||||
stream = true,
|
||||
model = %response_model,
|
||||
model = %envelope.model(),
|
||||
prompt_tokens = final_usage.prompt_token_count,
|
||||
output_tokens = final_usage.output_token_count,
|
||||
finish_reason = finish_reason.as_str(),
|
||||
@@ -456,16 +435,13 @@ async fn chat_completion_chunk_stream(
|
||||
);
|
||||
|
||||
if let Some(pending_chunk) = pending_chunk.as_mut()
|
||||
&& let Some(chunk) =
|
||||
pending_chunk.take_chunk(&request_id, &response_model, created)
|
||||
&& let Some(chunk) = pending_chunk.take_chunk(&envelope)
|
||||
{
|
||||
yield_chunk!(chunk);
|
||||
}
|
||||
|
||||
match final_chunk(
|
||||
&request_id,
|
||||
&response_model,
|
||||
created,
|
||||
&envelope,
|
||||
finish_reason,
|
||||
saw_tool_calls && !is_named_tool_choice,
|
||||
) {
|
||||
@@ -481,9 +457,7 @@ async fn chat_completion_chunk_stream(
|
||||
|
||||
if include_usage {
|
||||
y.yield_ok(usage_chunk(
|
||||
&request_id,
|
||||
&response_model,
|
||||
created,
|
||||
&envelope,
|
||||
Usage::from_token_usage(final_usage, enable_prompt_tokens_details),
|
||||
))
|
||||
.await;
|
||||
@@ -504,12 +478,10 @@ async fn chat_completion_chunk_stream(
|
||||
}
|
||||
|
||||
fn usage_chunk(
|
||||
request_id: &str,
|
||||
response_model: &str,
|
||||
created: u64,
|
||||
envelope: &Arc<StreamResponseEnvelope>,
|
||||
usage: Usage,
|
||||
) -> ChatCompletionStreamResponse {
|
||||
let mut chunk = ChatCompletionStreamResponse::new(request_id, response_model, created);
|
||||
let mut chunk = ChatCompletionStreamResponse::new(envelope);
|
||||
chunk.usage = Some(usage);
|
||||
chunk
|
||||
}
|
||||
@@ -588,9 +560,7 @@ impl PendingChatChunk {
|
||||
/// necessarily with a visible/chat-semantic delta.
|
||||
fn take_chunk(
|
||||
&mut self,
|
||||
request_id: &str,
|
||||
response_model: &str,
|
||||
created: u64,
|
||||
envelope: &Arc<StreamResponseEnvelope>,
|
||||
) -> Option<ChatCompletionStreamResponse> {
|
||||
let has_delta = self.delta.content.is_some()
|
||||
|| self.delta.reasoning.is_some()
|
||||
@@ -601,7 +571,7 @@ impl PendingChatChunk {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut chunk = ChatCompletionStreamResponse::new(request_id, response_model, created);
|
||||
let mut chunk = ChatCompletionStreamResponse::new(envelope);
|
||||
chunk.choices.push(ChatCompletionStreamChoice {
|
||||
delta: self.take_delta(),
|
||||
logprobs,
|
||||
@@ -659,18 +629,19 @@ async fn chat_completion_sse_stream(
|
||||
|
||||
/// Serialize one OpenAI chunk payload into one SSE `data:` event.
|
||||
fn to_sse_event(chunk: &ChatCompletionStreamResponse) -> Event {
|
||||
let payload =
|
||||
serde_json::to_string(chunk).expect("ChatCompletionStreamResponse must serialize to JSON");
|
||||
trace!(payload, "chat completion emitting chunk");
|
||||
Event::default().data(payload)
|
||||
trace!(?chunk, "chat completion emitting chunk");
|
||||
Event::default()
|
||||
.json_data(chunk)
|
||||
.expect("ChatCompletionStreamResponse must serialize to JSON")
|
||||
}
|
||||
|
||||
/// Serialize one OpenAI error payload into one SSE `data:` event.
|
||||
fn to_error_sse_event(error: &ApiError) -> Event {
|
||||
let payload = serde_json::to_string(&error.to_error_response())
|
||||
.expect("ErrorResponse must serialize to JSON");
|
||||
trace!(payload, "chat completion emitting error");
|
||||
Event::default().data(payload)
|
||||
let response = error.to_error_response();
|
||||
trace!(?response, "chat completion emitting error");
|
||||
Event::default()
|
||||
.json_data(response)
|
||||
.expect("ErrorResponse must serialize to JSON")
|
||||
}
|
||||
|
||||
/// Build the terminal OpenAI SSE sentinel event.
|
||||
@@ -681,12 +652,8 @@ fn done_sse_event() -> Event {
|
||||
|
||||
/// Build the initial assistant-role SSE chunk required by the OpenAI streaming
|
||||
/// protocol.
|
||||
fn start_chunk(
|
||||
request_id: &str,
|
||||
response_model: &str,
|
||||
created: u64,
|
||||
) -> ChatCompletionStreamResponse {
|
||||
let mut chunk = ChatCompletionStreamResponse::new(request_id, response_model, created);
|
||||
fn start_chunk(envelope: &Arc<StreamResponseEnvelope>) -> ChatCompletionStreamResponse {
|
||||
let mut chunk = ChatCompletionStreamResponse::new(envelope);
|
||||
chunk.choices.push(ChatCompletionStreamChoice {
|
||||
delta: ChatMessageDelta {
|
||||
role: Some(AssistantRole),
|
||||
@@ -699,9 +666,7 @@ fn start_chunk(
|
||||
|
||||
/// Build one content-delta SSE chunk from one internal assistant block delta.
|
||||
fn block_delta_chunk(
|
||||
request_id: &str,
|
||||
response_model: &str,
|
||||
created: u64,
|
||||
envelope: &Arc<StreamResponseEnvelope>,
|
||||
kind: AssistantBlockKind,
|
||||
delta: String,
|
||||
) -> ChatCompletionStreamResponse {
|
||||
@@ -719,7 +684,7 @@ fn block_delta_chunk(
|
||||
}
|
||||
};
|
||||
|
||||
let mut chunk = ChatCompletionStreamResponse::new(request_id, response_model, created);
|
||||
let mut chunk = ChatCompletionStreamResponse::new(envelope);
|
||||
chunk.choices.push(ChatCompletionStreamChoice {
|
||||
delta,
|
||||
..Default::default()
|
||||
@@ -728,14 +693,12 @@ fn block_delta_chunk(
|
||||
}
|
||||
|
||||
fn tool_call_start_chunk(
|
||||
request_id: &str,
|
||||
response_model: &str,
|
||||
created: u64,
|
||||
envelope: &Arc<StreamResponseEnvelope>,
|
||||
tool_index: u32,
|
||||
id: String,
|
||||
name: String,
|
||||
) -> ChatCompletionStreamResponse {
|
||||
let mut chunk = ChatCompletionStreamResponse::new(request_id, response_model, created);
|
||||
let mut chunk = ChatCompletionStreamResponse::new(envelope);
|
||||
chunk.choices.push(ChatCompletionStreamChoice {
|
||||
delta: ChatMessageDelta {
|
||||
tool_calls: Some(vec![ToolCallDelta {
|
||||
@@ -755,13 +718,11 @@ fn tool_call_start_chunk(
|
||||
}
|
||||
|
||||
fn tool_call_arguments_chunk(
|
||||
request_id: &str,
|
||||
response_model: &str,
|
||||
created: u64,
|
||||
envelope: &Arc<StreamResponseEnvelope>,
|
||||
tool_index: u32,
|
||||
delta: String,
|
||||
) -> ChatCompletionStreamResponse {
|
||||
let mut chunk = ChatCompletionStreamResponse::new(request_id, response_model, created);
|
||||
let mut chunk = ChatCompletionStreamResponse::new(envelope);
|
||||
chunk.choices.push(ChatCompletionStreamChoice {
|
||||
delta: ChatMessageDelta {
|
||||
tool_calls: Some(vec![ToolCallDelta {
|
||||
@@ -781,12 +742,10 @@ fn tool_call_arguments_chunk(
|
||||
}
|
||||
|
||||
fn logprobs_only_chunk(
|
||||
request_id: &str,
|
||||
response_model: &str,
|
||||
created: u64,
|
||||
envelope: &Arc<StreamResponseEnvelope>,
|
||||
logprobs: ChatLogProbs,
|
||||
) -> ChatCompletionStreamResponse {
|
||||
let mut chunk = ChatCompletionStreamResponse::new(request_id, response_model, created);
|
||||
let mut chunk = ChatCompletionStreamResponse::new(envelope);
|
||||
chunk.choices.push(ChatCompletionStreamChoice {
|
||||
logprobs: Some(logprobs),
|
||||
..Default::default()
|
||||
@@ -796,9 +755,7 @@ fn logprobs_only_chunk(
|
||||
|
||||
/// Build the terminal SSE chunk carrying the OpenAI finish reason.
|
||||
fn final_chunk(
|
||||
request_id: &str,
|
||||
response_model: &str,
|
||||
created: u64,
|
||||
envelope: &Arc<StreamResponseEnvelope>,
|
||||
finish_reason: FinishReason,
|
||||
use_tool_calls_finish_reason: bool,
|
||||
) -> Result<ChatCompletionStreamResponse, ApiError> {
|
||||
@@ -811,7 +768,7 @@ fn final_chunk(
|
||||
"chat stream finished"
|
||||
);
|
||||
|
||||
let mut chunk = ChatCompletionStreamResponse::new(request_id, response_model, created);
|
||||
let mut chunk = ChatCompletionStreamResponse::new(envelope);
|
||||
chunk.choices.push(ChatCompletionStreamChoice {
|
||||
finish_reason: Some(finish_reason.to_string()),
|
||||
stop_reason,
|
||||
@@ -844,6 +801,8 @@ fn stop_reason_to_json(stop_reason: &StopReason) -> Value {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use futures::{StreamExt as _, stream};
|
||||
use serde_json::json;
|
||||
use vllm_chat::{
|
||||
@@ -853,19 +812,23 @@ mod tests {
|
||||
use vllm_text::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTokenLogprob};
|
||||
|
||||
use super::{
|
||||
ApiServerOptions, ResponseOptions, block_delta_chunk, chat_completion_chunk_stream,
|
||||
final_chunk,
|
||||
ApiServerOptions, ResponseOptions, StreamResponseEnvelope, block_delta_chunk,
|
||||
chat_completion_chunk_stream, final_chunk,
|
||||
};
|
||||
|
||||
fn stream_envelope() -> Arc<StreamResponseEnvelope> {
|
||||
Arc::new(StreamResponseEnvelope::new(
|
||||
"chatcmpl-1".to_string(),
|
||||
"chat.completion.chunk",
|
||||
1,
|
||||
"model".to_string(),
|
||||
))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn text_chunk_uses_content_only_delta() {
|
||||
let chunk = block_delta_chunk(
|
||||
"chatcmpl-1",
|
||||
"model",
|
||||
1,
|
||||
AssistantBlockKind::Text,
|
||||
"hello".to_string(),
|
||||
);
|
||||
let envelope = stream_envelope();
|
||||
let chunk = block_delta_chunk(&envelope, AssistantBlockKind::Text, "hello".to_string());
|
||||
assert_eq!(chunk.choices[0].delta.role, None);
|
||||
assert_eq!(chunk.choices[0].delta.content.as_deref(), Some("hello"));
|
||||
assert_eq!(chunk.choices[0].delta.reasoning, None);
|
||||
@@ -873,10 +836,9 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn reasoning_chunk_uses_reasoning_only_delta() {
|
||||
let envelope = stream_envelope();
|
||||
let chunk = block_delta_chunk(
|
||||
"chatcmpl-1",
|
||||
"model",
|
||||
1,
|
||||
&envelope,
|
||||
AssistantBlockKind::Reasoning,
|
||||
"thinking".to_string(),
|
||||
);
|
||||
@@ -890,10 +852,9 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn final_chunk_maps_stop_finish_reason_and_stop_reason() {
|
||||
let envelope = stream_envelope();
|
||||
let chunk = final_chunk(
|
||||
"chatcmpl-1",
|
||||
"model",
|
||||
1,
|
||||
&envelope,
|
||||
FinishReason::Stop(Some(StopReason::Text("stop".to_string()))),
|
||||
false,
|
||||
)
|
||||
@@ -905,7 +866,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn final_chunk_maps_length_finish_reason() {
|
||||
let chunk = final_chunk("chatcmpl-1", "model", 1, FinishReason::Length, false)
|
||||
let chunk = final_chunk(&stream_envelope(), FinishReason::Length, false)
|
||||
.expect("finish reason is valid");
|
||||
|
||||
assert_eq!(chunk.choices[0].finish_reason.as_deref(), Some("length"));
|
||||
@@ -914,7 +875,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn final_chunk_maps_abort_finish_reason() {
|
||||
let chunk = final_chunk("chatcmpl-1", "model", 1, FinishReason::Abort, false)
|
||||
let chunk = final_chunk(&stream_envelope(), FinishReason::Abort, false)
|
||||
.expect("abort is a valid finish reason");
|
||||
|
||||
assert_eq!(chunk.choices[0].finish_reason.as_deref(), Some("abort"));
|
||||
@@ -923,12 +884,12 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn final_chunk_rejects_error_finish_reason() {
|
||||
assert!(final_chunk("chatcmpl-1", "model", 1, FinishReason::Error, false).is_err());
|
||||
assert!(final_chunk(&stream_envelope(), FinishReason::Error, false).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn final_chunk_maps_stop_to_tool_calls_when_tool_calls_were_streamed() {
|
||||
let chunk = final_chunk("chatcmpl-1", "model", 1, FinishReason::stop_eos(), true)
|
||||
let chunk = final_chunk(&stream_envelope(), FinishReason::stop_eos(), true)
|
||||
.expect("finish reason is valid");
|
||||
|
||||
assert_eq!(
|
||||
@@ -1002,6 +963,7 @@ mod tests {
|
||||
.expect("stream chunks");
|
||||
|
||||
assert_eq!(chunks.len(), 4);
|
||||
assert!(chunks.windows(2).all(|pair| Arc::ptr_eq(&pair[0].envelope, &pair[1].envelope)));
|
||||
assert_eq!(chunks[1].choices[0].delta.content.as_deref(), Some("hi"));
|
||||
let logprobs = chunks[1].choices[0].logprobs.as_ref().expect("logprobs");
|
||||
let content = logprobs.content.as_ref().expect("logprobs content");
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fmt;
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
@@ -13,9 +14,9 @@ use vllm_engine_core_client::protocol::sampling::RepetitionDetectionParams;
|
||||
|
||||
use crate::routes::openai::utils::structured_outputs::ResponseFormat;
|
||||
use crate::routes::openai::utils::types::{
|
||||
ChatLogProbs, ChatMessage, Normalizable, PromptLogprobs, StreamOptions, StringOrArray, Tool,
|
||||
ToolCall, ToolCallDelta, ToolChoice, Usage, default_true, deserialize_request_top_k,
|
||||
validate_messages, validate_stop, validate_top_p_value,
|
||||
ChatLogProbs, ChatMessage, Normalizable, PromptLogprobs, StreamOptions, StreamResponseEnvelope,
|
||||
StringOrArray, Tool, ToolCall, ToolCallDelta, ToolChoice, Usage, default_true,
|
||||
deserialize_request_top_k, validate_messages, validate_stop, validate_top_p_value,
|
||||
};
|
||||
|
||||
/// vLLM-compatible request type for the Chat Completions API.
|
||||
@@ -383,10 +384,8 @@ pub(super) struct ChatCompletionMessage {
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct ChatCompletionStreamResponse {
|
||||
pub id: String,
|
||||
pub object: String,
|
||||
pub created: u64,
|
||||
pub model: String,
|
||||
#[serde(flatten)]
|
||||
pub envelope: Arc<StreamResponseEnvelope>,
|
||||
pub choices: Vec<ChatCompletionStreamChoice>,
|
||||
pub usage: Option<Usage>,
|
||||
pub prompt_token_ids: Option<Vec<u32>>,
|
||||
@@ -394,12 +393,9 @@ pub(super) struct ChatCompletionStreamResponse {
|
||||
|
||||
impl ChatCompletionStreamResponse {
|
||||
/// Create a stream response with the standard envelope fields pre-filled.
|
||||
pub fn new(id: &str, model: &str, created: u64) -> Self {
|
||||
pub fn new(envelope: &Arc<StreamResponseEnvelope>) -> Self {
|
||||
Self {
|
||||
id: id.to_string(),
|
||||
object: "chat.completion.chunk".to_string(),
|
||||
created,
|
||||
model: model.to_string(),
|
||||
envelope: Arc::clone(envelope),
|
||||
choices: Vec::new(),
|
||||
usage: None,
|
||||
prompt_token_ids: None,
|
||||
|
||||
@@ -33,7 +33,7 @@ use super::utils::logprobs::{
|
||||
collected_logprobs_to_openai, decoded_logprobs_to_openai, decoded_prompt_logprobs_to_openai,
|
||||
prompt_logprobs_to_maps, text_len,
|
||||
};
|
||||
use super::utils::types::Usage;
|
||||
use super::utils::types::{StreamResponseEnvelope, Usage};
|
||||
use crate::config::ApiServerOptions;
|
||||
use crate::error::{ApiError, bail_server_error, server_error, text_submit_error};
|
||||
use crate::lora::LoraModelResolution;
|
||||
@@ -257,6 +257,12 @@ async fn completion_chunk_stream(
|
||||
mut y: TryYielder<CompletionSseChunk, ApiError>,
|
||||
) -> Result<(), ApiError> {
|
||||
pin_mut!(stream);
|
||||
let envelope = Arc::new(StreamResponseEnvelope::new(
|
||||
request_id,
|
||||
"text_completion",
|
||||
created,
|
||||
response_model,
|
||||
));
|
||||
let mut visible_text_len = 0_u32;
|
||||
let mut first_chunk = true;
|
||||
let mut continuous_usage = ContinuousUsage::default();
|
||||
@@ -292,13 +298,7 @@ async fn completion_chunk_stream(
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut chunk = delta_chunk(
|
||||
&request_id,
|
||||
&response_model,
|
||||
created,
|
||||
prompt.clone(),
|
||||
logprobs,
|
||||
);
|
||||
let mut chunk = delta_chunk(&envelope, prompt.clone(), logprobs);
|
||||
if return_token_ids && first_chunk {
|
||||
if let Some(choice) = chunk.choices.first_mut() {
|
||||
choice.prompt_token_ids = Some(prompt_token_ids.to_vec());
|
||||
@@ -308,8 +308,7 @@ async fn completion_chunk_stream(
|
||||
yield_chunk!(chunk);
|
||||
} else if return_token_ids {
|
||||
// Emit a chunk with prompt_token_ids in the first streaming response
|
||||
let mut chunk =
|
||||
delta_chunk(&request_id, &response_model, created, String::new(), None);
|
||||
let mut chunk = delta_chunk(&envelope, String::new(), None);
|
||||
if let Some(choice) = chunk.choices.first_mut() {
|
||||
choice.prompt_token_ids = Some(prompt_token_ids.to_vec());
|
||||
}
|
||||
@@ -331,7 +330,7 @@ async fn completion_chunk_stream(
|
||||
if enable_log_requests {
|
||||
info!(
|
||||
stream = true,
|
||||
model = %response_model,
|
||||
model = %envelope.model(),
|
||||
prompt_tokens = finished.usage.prompt_token_count,
|
||||
output_tokens = finished.usage.output_token_count,
|
||||
finish_reason = finished.finish_reason.as_str(),
|
||||
@@ -342,19 +341,12 @@ async fn completion_chunk_stream(
|
||||
finished.usage.prompt_token_count,
|
||||
finished.usage.output_token_count,
|
||||
);
|
||||
let final_chunk = final_chunk(
|
||||
&request_id,
|
||||
&response_model,
|
||||
created,
|
||||
finished.finish_reason,
|
||||
)?;
|
||||
let final_chunk = final_chunk(&envelope, finished.finish_reason)?;
|
||||
yield_chunk!(final_chunk);
|
||||
|
||||
if include_usage {
|
||||
y.yield_ok(CompletionSseChunk::Usage(usage_chunk(
|
||||
&request_id,
|
||||
&response_model,
|
||||
created,
|
||||
&envelope,
|
||||
Usage::from_token_usage(
|
||||
finished.usage,
|
||||
enable_prompt_tokens_details,
|
||||
@@ -380,7 +372,7 @@ async fn completion_chunk_stream(
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut chunk = delta_chunk(&request_id, &response_model, created, delta, logprobs);
|
||||
let mut chunk = delta_chunk(&envelope, delta, logprobs);
|
||||
let delta_token_count = token_ids.len();
|
||||
continuous_usage.add_output_tokens(delta_token_count);
|
||||
if return_token_ids && let Some(choice) = chunk.choices.first_mut() {
|
||||
@@ -393,7 +385,7 @@ async fn completion_chunk_stream(
|
||||
if enable_log_requests {
|
||||
info!(
|
||||
stream = true,
|
||||
model = %response_model,
|
||||
model = %envelope.model(),
|
||||
prompt_tokens = finished.usage.prompt_token_count,
|
||||
output_tokens = finished.usage.output_token_count,
|
||||
finish_reason = finished.finish_reason.as_str(),
|
||||
@@ -404,19 +396,12 @@ async fn completion_chunk_stream(
|
||||
finished.usage.prompt_token_count,
|
||||
finished.usage.output_token_count,
|
||||
);
|
||||
let final_chunk = final_chunk(
|
||||
&request_id,
|
||||
&response_model,
|
||||
created,
|
||||
finished.finish_reason,
|
||||
)?;
|
||||
let final_chunk = final_chunk(&envelope, finished.finish_reason)?;
|
||||
yield_chunk!(final_chunk);
|
||||
|
||||
if include_usage {
|
||||
y.yield_ok(CompletionSseChunk::Usage(usage_chunk(
|
||||
&request_id,
|
||||
&response_model,
|
||||
created,
|
||||
&envelope,
|
||||
Usage::from_token_usage(finished.usage, enable_prompt_tokens_details),
|
||||
)))
|
||||
.await;
|
||||
@@ -436,13 +421,11 @@ async fn completion_chunk_stream(
|
||||
}
|
||||
|
||||
fn delta_chunk(
|
||||
request_id: &str,
|
||||
response_model: &str,
|
||||
created: u64,
|
||||
envelope: &Arc<StreamResponseEnvelope>,
|
||||
text: String,
|
||||
logprobs: Option<LogProbs>,
|
||||
) -> CompletionStreamResponse {
|
||||
let mut chunk = CompletionStreamResponse::new(request_id, response_model, created);
|
||||
let mut chunk = CompletionStreamResponse::new(envelope);
|
||||
chunk.choices.push(CompletionStreamChoice {
|
||||
text,
|
||||
logprobs,
|
||||
@@ -452,15 +435,13 @@ fn delta_chunk(
|
||||
}
|
||||
|
||||
fn final_chunk(
|
||||
request_id: &str,
|
||||
response_model: &str,
|
||||
created: u64,
|
||||
envelope: &Arc<StreamResponseEnvelope>,
|
||||
finish_reason: FinishReason,
|
||||
) -> Result<CompletionStreamResponse, ApiError> {
|
||||
let stop_reason = finish_reason.as_stop_reason().map(stop_reason_to_json);
|
||||
let finish_reason = completion_finish_reason_to_openai(&finish_reason)?;
|
||||
|
||||
let mut chunk = CompletionStreamResponse::new(request_id, response_model, created);
|
||||
let mut chunk = CompletionStreamResponse::new(envelope);
|
||||
chunk.choices.push(CompletionStreamChoice {
|
||||
finish_reason: Some(finish_reason.to_string()),
|
||||
stop_reason,
|
||||
@@ -517,13 +498,8 @@ fn prompt_only_logprobs_to_openai(
|
||||
))
|
||||
}
|
||||
|
||||
fn usage_chunk(
|
||||
request_id: &str,
|
||||
response_model: &str,
|
||||
created: u64,
|
||||
usage: Usage,
|
||||
) -> CompletionStreamResponse {
|
||||
let mut chunk = CompletionStreamResponse::new(request_id, response_model, created);
|
||||
fn usage_chunk(envelope: &Arc<StreamResponseEnvelope>, usage: Usage) -> CompletionStreamResponse {
|
||||
let mut chunk = CompletionStreamResponse::new(envelope);
|
||||
chunk.usage = Some(usage);
|
||||
chunk
|
||||
}
|
||||
@@ -556,17 +532,19 @@ async fn completion_sse_stream(
|
||||
|
||||
/// Serialize one OpenAI chunk payload into one SSE `data:` event.
|
||||
fn to_sse_event(chunk: &CompletionSseChunk) -> Event {
|
||||
let payload = serde_json::to_string(chunk).expect("completion chunk must serialize to JSON");
|
||||
trace!(payload, "completion emitting chunk");
|
||||
Event::default().data(payload)
|
||||
trace!(?chunk, "completion emitting chunk");
|
||||
Event::default()
|
||||
.json_data(chunk)
|
||||
.expect("completion chunk must serialize to JSON")
|
||||
}
|
||||
|
||||
/// Serialize one OpenAI error payload into one SSE `data:` event.
|
||||
fn to_error_sse_event(error: &ApiError) -> Event {
|
||||
let payload = serde_json::to_string(&error.to_error_response())
|
||||
.expect("ErrorResponse must serialize to JSON");
|
||||
trace!(payload, "completion emitting error");
|
||||
Event::default().data(payload)
|
||||
let response = error.to_error_response();
|
||||
trace!(?response, "completion emitting error");
|
||||
Event::default()
|
||||
.json_data(response)
|
||||
.expect("ErrorResponse must serialize to JSON")
|
||||
}
|
||||
|
||||
/// Build the terminal OpenAI SSE sentinel event.
|
||||
@@ -577,6 +555,8 @@ fn done_sse_event() -> Event {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use futures::{StreamExt as _, stream};
|
||||
use itertools::Itertools as _;
|
||||
use vllm_engine_core_client::protocol::output::StopReason;
|
||||
@@ -586,13 +566,29 @@ mod tests {
|
||||
};
|
||||
|
||||
use super::{
|
||||
ApiServerOptions, CompletionSseChunk, ResponseOptions, completion_chunk_stream, final_chunk,
|
||||
ApiServerOptions, CompletionSseChunk, CompletionStreamResponse, ResponseOptions,
|
||||
StreamResponseEnvelope, completion_chunk_stream, final_chunk,
|
||||
};
|
||||
|
||||
fn stream_envelope() -> Arc<StreamResponseEnvelope> {
|
||||
Arc::new(StreamResponseEnvelope::new(
|
||||
"cmpl-1".to_string(),
|
||||
"text_completion",
|
||||
1,
|
||||
"model".to_string(),
|
||||
))
|
||||
}
|
||||
|
||||
fn chunk_response(chunk: &CompletionSseChunk) -> &CompletionStreamResponse {
|
||||
match chunk {
|
||||
CompletionSseChunk::Chunk(response) | CompletionSseChunk::Usage(response) => response,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn final_chunk_maps_stop_finish_reason() {
|
||||
let chunk = final_chunk("cmpl-1", "model", 1, FinishReason::stop_eos())
|
||||
.expect("finish reason valid");
|
||||
let chunk =
|
||||
final_chunk(&stream_envelope(), FinishReason::stop_eos()).expect("finish reason valid");
|
||||
assert_eq!(chunk.choices[0].finish_reason.as_deref(), Some("stop"));
|
||||
assert_eq!(chunk.choices[0].text, "");
|
||||
}
|
||||
@@ -600,20 +596,20 @@ mod tests {
|
||||
#[test]
|
||||
fn final_chunk_maps_length_finish_reason() {
|
||||
let chunk =
|
||||
final_chunk("cmpl-1", "model", 1, FinishReason::Length).expect("finish reason valid");
|
||||
final_chunk(&stream_envelope(), FinishReason::Length).expect("finish reason valid");
|
||||
assert_eq!(chunk.choices[0].finish_reason.as_deref(), Some("length"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn final_chunk_maps_abort_finish_reason() {
|
||||
let chunk =
|
||||
final_chunk("cmpl-1", "model", 1, FinishReason::Abort).expect("finish reason valid");
|
||||
final_chunk(&stream_envelope(), FinishReason::Abort).expect("finish reason valid");
|
||||
assert_eq!(chunk.choices[0].finish_reason.as_deref(), Some("abort"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn final_chunk_rejects_error_finish_reason() {
|
||||
assert!(final_chunk("cmpl-1", "model", 1, FinishReason::Error).is_err());
|
||||
assert!(final_chunk(&stream_envelope(), FinishReason::Error).is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -702,6 +698,13 @@ mod tests {
|
||||
|
||||
let chunks: Vec<_> = chunks.into_iter().try_collect().expect("stream should succeed");
|
||||
|
||||
assert!(chunks.windows(2).all(|pair| {
|
||||
Arc::ptr_eq(
|
||||
&chunk_response(&pair[0]).envelope,
|
||||
&chunk_response(&pair[1]).envelope,
|
||||
)
|
||||
}));
|
||||
|
||||
match &chunks[0] {
|
||||
CompletionSseChunk::Chunk(chunk) => {
|
||||
assert_eq!(chunk.choices[0].text, "h");
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
@@ -10,8 +11,8 @@ use vllm_engine_core_client::protocol::sampling::RepetitionDetectionParams;
|
||||
use vllm_text::Prompt;
|
||||
|
||||
use crate::routes::openai::utils::types::{
|
||||
LogProbs, Normalizable, PromptLogprobs, StreamOptions, StringOrArray, Usage, default_true,
|
||||
deserialize_request_top_k, validate_stop,
|
||||
LogProbs, Normalizable, PromptLogprobs, StreamOptions, StreamResponseEnvelope, StringOrArray,
|
||||
Usage, default_true, deserialize_request_top_k, validate_stop,
|
||||
};
|
||||
|
||||
/// Serde default for `CompletionRequest::max_tokens`, matching the Python vLLM
|
||||
@@ -241,22 +242,17 @@ pub(super) struct CompletionChoice {
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct CompletionStreamResponse {
|
||||
pub id: String,
|
||||
pub object: String,
|
||||
pub created: u64,
|
||||
pub model: String,
|
||||
#[serde(flatten)]
|
||||
pub envelope: Arc<StreamResponseEnvelope>,
|
||||
pub choices: Vec<CompletionStreamChoice>,
|
||||
pub usage: Option<Usage>,
|
||||
}
|
||||
|
||||
impl CompletionStreamResponse {
|
||||
/// Create a stream response with the standard envelope fields pre-filled.
|
||||
pub fn new(id: &str, model: &str, created: u64) -> Self {
|
||||
pub fn new(envelope: &Arc<StreamResponseEnvelope>) -> Self {
|
||||
Self {
|
||||
id: id.to_string(),
|
||||
object: "text_completion".to_string(),
|
||||
created,
|
||||
model: model.to_string(),
|
||||
envelope: Arc::clone(envelope),
|
||||
choices: Vec::new(),
|
||||
usage: None,
|
||||
}
|
||||
|
||||
@@ -201,6 +201,29 @@ pub struct InputAudio {
|
||||
// Streaming
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub(crate) struct StreamResponseEnvelope {
|
||||
id: String,
|
||||
object: &'static str,
|
||||
created: u64,
|
||||
model: String,
|
||||
}
|
||||
|
||||
impl StreamResponseEnvelope {
|
||||
pub(crate) fn new(id: String, object: &'static str, created: u64, model: String) -> Self {
|
||||
Self {
|
||||
id,
|
||||
object,
|
||||
created,
|
||||
model,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn model(&self) -> &str {
|
||||
&self.model
|
||||
}
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `StreamOptions` class.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct StreamOptions {
|
||||
|
||||
@@ -170,6 +170,22 @@ fn sse_json_payloads(text: &str) -> Vec<serde_json::Value> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn assert_shared_openai_stream_envelope(payloads: &[serde_json::Value], object: &str, model: &str) {
|
||||
let first = payloads.first().expect("at least one SSE JSON payload");
|
||||
assert!(first["id"].as_str().is_some_and(|id| !id.is_empty()));
|
||||
assert!(first["created"].is_u64());
|
||||
assert_eq!(first["object"], object);
|
||||
assert_eq!(first["model"], model);
|
||||
|
||||
for payload in payloads {
|
||||
assert_eq!(payload["id"], first["id"]);
|
||||
assert_eq!(payload["object"], first["object"]);
|
||||
assert_eq!(payload["created"], first["created"]);
|
||||
assert_eq!(payload["model"], first["model"]);
|
||||
assert!(payload.get("envelope").is_none());
|
||||
}
|
||||
}
|
||||
|
||||
type TestFuture<'a> = Pin<Box<dyn Future<Output = ()> + Send + 'a>>;
|
||||
|
||||
fn boxed_test_future<'a>(future: impl Future<Output = ()> + Send + 'a) -> TestFuture<'a> {
|
||||
@@ -2717,6 +2733,11 @@ async fn happy_path_returns_sse_stream() {
|
||||
let text = String::from_utf8(body.to_vec()).expect("utf8 body");
|
||||
let after = METRICS.render().unwrap();
|
||||
|
||||
assert_shared_openai_stream_envelope(
|
||||
&sse_json_payloads(&text),
|
||||
"chat.completion.chunk",
|
||||
"Qwen/Qwen1.5-0.5B-Chat",
|
||||
);
|
||||
assert!(text.contains("\"role\":\"assistant\""), "{text}");
|
||||
assert!(text.starts_with("data: "), "{text}");
|
||||
assert_eq!(
|
||||
@@ -4465,6 +4486,11 @@ async fn completions_happy_path_returns_sse_stream() {
|
||||
engine_task.await.expect("mock engine task");
|
||||
let text = String::from_utf8(body.to_vec()).expect("utf8 body");
|
||||
let payloads = sse_data_payloads(&text);
|
||||
assert_shared_openai_stream_envelope(
|
||||
&sse_json_payloads(&text),
|
||||
"text_completion",
|
||||
"Qwen/Qwen1.5-0.5B-Chat",
|
||||
);
|
||||
let usage_index = payloads
|
||||
.iter()
|
||||
.position(|payload| payload.contains("\"usage\":"))
|
||||
|
||||
Reference in New Issue
Block a user