[Rust Frontend] Support --sse-keep-alive-interval (#58306)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-09-24 06:19:14 +00:00
committed by GitHub
co-authored by Claude Opus 5.5 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent cf78797cb7
commit e028554a1f
7 changed files with 135 additions and 10 deletions
+10
View File
@@ -363,6 +363,14 @@ pub struct SharedRuntimeArgs {
#[serde(default)]
pub enable_scale_out: bool,
/// Send an SSE keep-alive comment line every this many seconds when a
/// streaming response is idle (queued, prefill, or between tokens), to
/// prevent reverse proxies/tunnels with read timeouts from closing the
/// connection. Defaults to 0, which disables keep-alive comments entirely.
#[arg(long, default_value_t = 0)]
#[serde(default)]
pub sse_keep_alive_interval: u64,
/// If provided, the server will require one of these keys to be presented
/// in the Authorization header.
#[educe(Debug(ignore))]
@@ -616,6 +624,8 @@ impl SharedRuntimeArgs {
enable_prompt_tokens_details: self.enable_prompt_tokens_details,
enable_request_id_headers: self.enable_request_id_headers,
enable_scale_out: self.enable_scale_out,
sse_keep_alive_interval: (self.sse_keep_alive_interval > 0)
.then(|| Duration::from_secs(self.sse_keep_alive_interval)),
}
}
+6
View File
@@ -200,6 +200,7 @@ fn serve_args_forward_python_flags_with_separator() {
enable_prompt_tokens_details: false,
enable_request_id_headers: false,
enable_scale_out: false,
sse_keep_alive_interval: 0,
disable_log_stats: false,
served_model_name: [],
allowed_origins: JsonStringList(
@@ -1040,6 +1041,7 @@ fn frontend_args_accept_json() {
enable_prompt_tokens_details: false,
enable_request_id_headers: false,
enable_scale_out: false,
sse_keep_alive_interval: 0,
disable_log_stats: false,
served_model_name: [],
allowed_origins: JsonStringList(
@@ -1711,6 +1713,7 @@ fn serve_args_accept_handshake_aliases() {
enable_prompt_tokens_details: false,
enable_request_id_headers: false,
enable_scale_out: false,
sse_keep_alive_interval: 0,
disable_log_stats: false,
served_model_name: [],
allowed_origins: JsonStringList(
@@ -1867,6 +1870,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
enable_prompt_tokens_details: false,
enable_request_id_headers: false,
enable_scale_out: false,
sse_keep_alive_interval: None,
},
cors: CorsConfig {
allow_origins: [
@@ -1960,6 +1964,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
enable_prompt_tokens_details: false,
enable_request_id_headers: false,
enable_scale_out: false,
sse_keep_alive_interval: None,
},
cors: CorsConfig {
allow_origins: [
@@ -2074,6 +2079,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
enable_prompt_tokens_details: false,
enable_request_id_headers: false,
enable_scale_out: false,
sse_keep_alive_interval: None,
},
cors: CorsConfig {
allow_origins: [
+3
View File
@@ -59,6 +59,9 @@ pub struct ApiServerOptions {
pub enable_request_id_headers: bool,
/// When `true`, register the scale-out `/inference/v1/generate` route.
pub enable_scale_out: bool,
/// Idle interval after which streaming SSE responses send a keep-alive
/// comment. `None` disables keep-alive comments.
pub sse_keep_alive_interval: Option<Duration>,
}
/// CORS settings mirroring Python's `CORSMiddleware`; the default is permissive.
@@ -14,7 +14,7 @@ use asynk_strim_attr::{TryYielder, try_stream};
use axum::Json;
use axum::extract::State;
use axum::http::HeaderMap;
use axum::response::sse::{Event, Sse};
use axum::response::sse::Event;
use axum::response::{IntoResponse, Response};
use futures::{Stream, StreamExt as _, pin_mut};
use thiserror_ext::AsReport as _;
@@ -40,7 +40,7 @@ use crate::routes::openai::utils::logprobs::clamp_logprob;
use crate::routes::openai::utils::types::{ChatLogProbs, ChatLogProbsContent, TopLogProb, Usage};
use crate::routes::openai::utils::validated_json::ValidatedJson;
use crate::state::AppState;
use crate::utils::resolve_request_context;
use crate::utils::{resolve_request_context, sse_response};
/// Validate one token-in/token-out request and proxy it into the shared
/// `vllm-text` stack.
@@ -114,7 +114,7 @@ pub async fn generate(
);
let sse_stream = generate_sse_stream(chunk_stream).instrument(request_span);
return Sse::new(sse_stream).into_response();
return sse_response(sse_stream, api_server_options.sse_keep_alive_interval);
}
let collected = match raw_stream.collect_output().instrument(request_span.clone()).await {
@@ -13,7 +13,7 @@ use asynk_strim_attr::{TryYielder, try_stream};
use axum::Json;
use axum::extract::State;
use axum::http::HeaderMap;
use axum::response::sse::{Event, Sse};
use axum::response::sse::Event;
use axum::response::{IntoResponse, Response};
use futures::{Stream, StreamExt as _, pin_mut};
use serde_json::Value;
@@ -45,7 +45,7 @@ use crate::routes::openai::utils::types::{
use crate::routes::openai::utils::usage::ContinuousUsage;
use crate::routes::openai::utils::validated_json::ValidatedJson;
use crate::state::AppState;
use crate::utils::{ResolvedRequestContext, resolve_request_context, unix_timestamp};
use crate::utils::{ResolvedRequestContext, resolve_request_context, sse_response, unix_timestamp};
pub(crate) fn lower_chat_request(
request: ChatCompletionRequest,
@@ -99,7 +99,7 @@ pub async fn chat_completions(
);
let sse_stream = chat_completion_sse_stream(chunk_stream).instrument(request_span);
Sse::new(sse_stream).into_response()
sse_response(sse_stream, api_server_options.sse_keep_alive_interval)
} else {
let response = match collect_chat_completion(
chat_stream,
@@ -13,7 +13,7 @@ use asynk_strim_attr::{TryYielder, try_stream};
use axum::Json;
use axum::extract::State;
use axum::http::HeaderMap;
use axum::response::sse::{Event, Sse};
use axum::response::sse::Event;
use axum::response::{IntoResponse, Response};
use futures::{Stream, StreamExt as _, pin_mut};
use serde_json::Value;
@@ -45,7 +45,7 @@ use crate::routes::openai::utils::types::LogProbs;
use crate::routes::openai::utils::usage::ContinuousUsage;
use crate::routes::openai::utils::validated_json::ValidatedJson;
use crate::state::AppState;
use crate::utils::{ResolvedRequestContext, resolve_request_context, unix_timestamp};
use crate::utils::{ResolvedRequestContext, resolve_request_context, sse_response, unix_timestamp};
pub(crate) fn lower_completion_request(
request: CompletionRequest,
@@ -111,7 +111,7 @@ pub async fn completions(
);
let sse_stream = completion_sse_stream(chunk_stream).instrument(request_span);
Sse::new(sse_stream).into_response()
sse_response(sse_stream, api_server_options.sse_keep_alive_interval)
} else {
let response = match collect_completion(
text_stream,
+107 -1
View File
@@ -2,9 +2,13 @@
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::collections::HashMap;
use std::time::{SystemTime, UNIX_EPOCH};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use axum::BoxError;
use axum::http::HeaderMap;
use axum::response::sse::{Event, KeepAlive, Sse};
use axum::response::{IntoResponse, Response};
use futures::Stream;
use serde_json::Value;
use thiserror_ext::AsReport;
use uuid::Uuid;
@@ -27,6 +31,27 @@ pub fn unix_timestamp() -> u64 {
.unwrap_or_default()
}
/// Build a streaming SSE response, optionally sending keep-alive comments
/// while the stream is idle.
///
/// With `keep_alive_interval` set, a `: keep-alive` comment is sent whenever no
/// event has been produced for that long (queued, prefill, between tokens, or
/// while a parser buffers a long tool call). SSE parsers ignore comments, but
/// they still count as bytes for read timeouts in reverse proxies and clients.
pub fn sse_response<S, E>(stream: S, keep_alive_interval: Option<Duration>) -> Response
where
S: Stream<Item = Result<Event, E>> + Send + 'static,
E: Into<BoxError>,
{
let sse = Sse::new(stream);
match keep_alive_interval {
Some(interval) => sse
.keep_alive(KeepAlive::new().interval(interval).text("keep-alive"))
.into_response(),
None => sse.into_response(),
}
}
/// Construct an API error for a failed utility call to the engine core.
pub fn utility_call_error(method: &str, error: impl AsReport) -> ApiError {
ApiError::server_error(format!("failed to call {method}: {}", error.as_report()))
@@ -157,3 +182,84 @@ pub fn resolve_base_request_id(
id
})
}
#[cfg(test)]
mod tests {
use std::convert::Infallible;
use futures::{StreamExt as _, stream};
use tokio::time::{Instant, sleep};
use super::*;
/// Collect `(elapsed seconds, frame)` pairs from a response whose event
/// stream idles for 20 seconds between its two events.
async fn idle_stream_frames(keep_alive_interval: Option<Duration>) -> Vec<(u64, String)> {
let events = stream::iter([Ok::<_, Infallible>(Event::default().data("first"))]).chain(
stream::once(async {
sleep(Duration::from_secs(20)).await;
Ok(Event::default().data("second"))
}),
);
let start = Instant::now();
sse_response(events, keep_alive_interval)
.into_body()
.into_data_stream()
.map(|frame| {
let frame = frame.expect("read frame");
(
start.elapsed().as_secs(),
String::from_utf8(frame.to_vec()).expect("utf-8 frame"),
)
})
.collect()
.await
}
/// An idle gap is filled with `: keep-alive` comments at the configured
/// interval, and real events still pass through in order.
#[tokio::test(start_paused = true)]
async fn sse_response_sends_keep_alive_comments_while_idle() {
let frames = idle_stream_frames(Some(Duration::from_secs(8))).await;
expect_test::expect![[r#"
[
(
0,
"data: first\n\n",
),
(
8,
": keep-alive\n\n",
),
(
16,
": keep-alive\n\n",
),
(
20,
"data: second\n\n",
),
]
"#]]
.assert_debug_eq(&frames);
}
/// Without an interval, an idle gap produces no extra bytes.
#[tokio::test(start_paused = true)]
async fn sse_response_without_keep_alive_interval_sends_no_comments() {
let frames = idle_stream_frames(None).await;
expect_test::expect![[r#"
[
(
0,
"data: first\n\n",
),
(
20,
"data: second\n\n",
),
]
"#]]
.assert_debug_eq(&frames);
}
}