[Rust Frontend] Support --sse-keep-alive-interval (#58306)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
cf78797cb7
commit
e028554a1f
@@ -59,6 +59,9 @@ pub struct ApiServerOptions {
|
||||
pub enable_request_id_headers: bool,
|
||||
/// When `true`, register the scale-out `/inference/v1/generate` route.
|
||||
pub enable_scale_out: bool,
|
||||
/// Idle interval after which streaming SSE responses send a keep-alive
|
||||
/// comment. `None` disables keep-alive comments.
|
||||
pub sse_keep_alive_interval: Option<Duration>,
|
||||
}
|
||||
|
||||
/// CORS settings mirroring Python's `CORSMiddleware`; the default is permissive.
|
||||
|
||||
@@ -14,7 +14,7 @@ use asynk_strim_attr::{TryYielder, try_stream};
|
||||
use axum::Json;
|
||||
use axum::extract::State;
|
||||
use axum::http::HeaderMap;
|
||||
use axum::response::sse::{Event, Sse};
|
||||
use axum::response::sse::Event;
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use futures::{Stream, StreamExt as _, pin_mut};
|
||||
use thiserror_ext::AsReport as _;
|
||||
@@ -40,7 +40,7 @@ use crate::routes::openai::utils::logprobs::clamp_logprob;
|
||||
use crate::routes::openai::utils::types::{ChatLogProbs, ChatLogProbsContent, TopLogProb, Usage};
|
||||
use crate::routes::openai::utils::validated_json::ValidatedJson;
|
||||
use crate::state::AppState;
|
||||
use crate::utils::resolve_request_context;
|
||||
use crate::utils::{resolve_request_context, sse_response};
|
||||
|
||||
/// Validate one token-in/token-out request and proxy it into the shared
|
||||
/// `vllm-text` stack.
|
||||
@@ -114,7 +114,7 @@ pub async fn generate(
|
||||
);
|
||||
let sse_stream = generate_sse_stream(chunk_stream).instrument(request_span);
|
||||
|
||||
return Sse::new(sse_stream).into_response();
|
||||
return sse_response(sse_stream, api_server_options.sse_keep_alive_interval);
|
||||
}
|
||||
|
||||
let collected = match raw_stream.collect_output().instrument(request_span.clone()).await {
|
||||
|
||||
@@ -13,7 +13,7 @@ use asynk_strim_attr::{TryYielder, try_stream};
|
||||
use axum::Json;
|
||||
use axum::extract::State;
|
||||
use axum::http::HeaderMap;
|
||||
use axum::response::sse::{Event, Sse};
|
||||
use axum::response::sse::Event;
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use futures::{Stream, StreamExt as _, pin_mut};
|
||||
use serde_json::Value;
|
||||
@@ -45,7 +45,7 @@ use crate::routes::openai::utils::types::{
|
||||
use crate::routes::openai::utils::usage::ContinuousUsage;
|
||||
use crate::routes::openai::utils::validated_json::ValidatedJson;
|
||||
use crate::state::AppState;
|
||||
use crate::utils::{ResolvedRequestContext, resolve_request_context, unix_timestamp};
|
||||
use crate::utils::{ResolvedRequestContext, resolve_request_context, sse_response, unix_timestamp};
|
||||
|
||||
pub(crate) fn lower_chat_request(
|
||||
request: ChatCompletionRequest,
|
||||
@@ -99,7 +99,7 @@ pub async fn chat_completions(
|
||||
);
|
||||
let sse_stream = chat_completion_sse_stream(chunk_stream).instrument(request_span);
|
||||
|
||||
Sse::new(sse_stream).into_response()
|
||||
sse_response(sse_stream, api_server_options.sse_keep_alive_interval)
|
||||
} else {
|
||||
let response = match collect_chat_completion(
|
||||
chat_stream,
|
||||
|
||||
@@ -13,7 +13,7 @@ use asynk_strim_attr::{TryYielder, try_stream};
|
||||
use axum::Json;
|
||||
use axum::extract::State;
|
||||
use axum::http::HeaderMap;
|
||||
use axum::response::sse::{Event, Sse};
|
||||
use axum::response::sse::Event;
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use futures::{Stream, StreamExt as _, pin_mut};
|
||||
use serde_json::Value;
|
||||
@@ -45,7 +45,7 @@ use crate::routes::openai::utils::types::LogProbs;
|
||||
use crate::routes::openai::utils::usage::ContinuousUsage;
|
||||
use crate::routes::openai::utils::validated_json::ValidatedJson;
|
||||
use crate::state::AppState;
|
||||
use crate::utils::{ResolvedRequestContext, resolve_request_context, unix_timestamp};
|
||||
use crate::utils::{ResolvedRequestContext, resolve_request_context, sse_response, unix_timestamp};
|
||||
|
||||
pub(crate) fn lower_completion_request(
|
||||
request: CompletionRequest,
|
||||
@@ -111,7 +111,7 @@ pub async fn completions(
|
||||
);
|
||||
let sse_stream = completion_sse_stream(chunk_stream).instrument(request_span);
|
||||
|
||||
Sse::new(sse_stream).into_response()
|
||||
sse_response(sse_stream, api_server_options.sse_keep_alive_interval)
|
||||
} else {
|
||||
let response = match collect_completion(
|
||||
text_stream,
|
||||
|
||||
@@ -2,9 +2,13 @@
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use axum::BoxError;
|
||||
use axum::http::HeaderMap;
|
||||
use axum::response::sse::{Event, KeepAlive, Sse};
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use futures::Stream;
|
||||
use serde_json::Value;
|
||||
use thiserror_ext::AsReport;
|
||||
use uuid::Uuid;
|
||||
@@ -27,6 +31,27 @@ pub fn unix_timestamp() -> u64 {
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Build a streaming SSE response, optionally sending keep-alive comments
|
||||
/// while the stream is idle.
|
||||
///
|
||||
/// With `keep_alive_interval` set, a `: keep-alive` comment is sent whenever no
|
||||
/// event has been produced for that long (queued, prefill, between tokens, or
|
||||
/// while a parser buffers a long tool call). SSE parsers ignore comments, but
|
||||
/// they still count as bytes for read timeouts in reverse proxies and clients.
|
||||
pub fn sse_response<S, E>(stream: S, keep_alive_interval: Option<Duration>) -> Response
|
||||
where
|
||||
S: Stream<Item = Result<Event, E>> + Send + 'static,
|
||||
E: Into<BoxError>,
|
||||
{
|
||||
let sse = Sse::new(stream);
|
||||
match keep_alive_interval {
|
||||
Some(interval) => sse
|
||||
.keep_alive(KeepAlive::new().interval(interval).text("keep-alive"))
|
||||
.into_response(),
|
||||
None => sse.into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Construct an API error for a failed utility call to the engine core.
|
||||
pub fn utility_call_error(method: &str, error: impl AsReport) -> ApiError {
|
||||
ApiError::server_error(format!("failed to call {method}: {}", error.as_report()))
|
||||
@@ -157,3 +182,84 @@ pub fn resolve_base_request_id(
|
||||
id
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::convert::Infallible;
|
||||
|
||||
use futures::{StreamExt as _, stream};
|
||||
use tokio::time::{Instant, sleep};
|
||||
|
||||
use super::*;
|
||||
|
||||
/// Collect `(elapsed seconds, frame)` pairs from a response whose event
|
||||
/// stream idles for 20 seconds between its two events.
|
||||
async fn idle_stream_frames(keep_alive_interval: Option<Duration>) -> Vec<(u64, String)> {
|
||||
let events = stream::iter([Ok::<_, Infallible>(Event::default().data("first"))]).chain(
|
||||
stream::once(async {
|
||||
sleep(Duration::from_secs(20)).await;
|
||||
Ok(Event::default().data("second"))
|
||||
}),
|
||||
);
|
||||
let start = Instant::now();
|
||||
sse_response(events, keep_alive_interval)
|
||||
.into_body()
|
||||
.into_data_stream()
|
||||
.map(|frame| {
|
||||
let frame = frame.expect("read frame");
|
||||
(
|
||||
start.elapsed().as_secs(),
|
||||
String::from_utf8(frame.to_vec()).expect("utf-8 frame"),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
.await
|
||||
}
|
||||
|
||||
/// An idle gap is filled with `: keep-alive` comments at the configured
|
||||
/// interval, and real events still pass through in order.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn sse_response_sends_keep_alive_comments_while_idle() {
|
||||
let frames = idle_stream_frames(Some(Duration::from_secs(8))).await;
|
||||
expect_test::expect![[r#"
|
||||
[
|
||||
(
|
||||
0,
|
||||
"data: first\n\n",
|
||||
),
|
||||
(
|
||||
8,
|
||||
": keep-alive\n\n",
|
||||
),
|
||||
(
|
||||
16,
|
||||
": keep-alive\n\n",
|
||||
),
|
||||
(
|
||||
20,
|
||||
"data: second\n\n",
|
||||
),
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&frames);
|
||||
}
|
||||
|
||||
/// Without an interval, an idle gap produces no extra bytes.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn sse_response_without_keep_alive_interval_sends_no_comments() {
|
||||
let frames = idle_stream_frames(None).await;
|
||||
expect_test::expect![[r#"
|
||||
[
|
||||
(
|
||||
0,
|
||||
"data: first\n\n",
|
||||
),
|
||||
(
|
||||
20,
|
||||
"data: second\n\n",
|
||||
),
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&frames);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user