diff --git a/rust/src/cmd/src/cli.rs b/rust/src/cmd/src/cli.rs index 5bdf66da57..c5a7c75baf 100644 --- a/rust/src/cmd/src/cli.rs +++ b/rust/src/cmd/src/cli.rs @@ -363,6 +363,14 @@ pub struct SharedRuntimeArgs { #[serde(default)] pub enable_scale_out: bool, + /// Send an SSE keep-alive comment line every this many seconds when a + /// streaming response is idle (queued, prefill, or between tokens), to + /// prevent reverse proxies/tunnels with read timeouts from closing the + /// connection. Defaults to 0, which disables keep-alive comments entirely. + #[arg(long, default_value_t = 0)] + #[serde(default)] + pub sse_keep_alive_interval: u64, + /// If provided, the server will require one of these keys to be presented /// in the Authorization header. #[educe(Debug(ignore))] @@ -616,6 +624,8 @@ impl SharedRuntimeArgs { enable_prompt_tokens_details: self.enable_prompt_tokens_details, enable_request_id_headers: self.enable_request_id_headers, enable_scale_out: self.enable_scale_out, + sse_keep_alive_interval: (self.sse_keep_alive_interval > 0) + .then(|| Duration::from_secs(self.sse_keep_alive_interval)), } } diff --git a/rust/src/cmd/src/cli/tests.rs b/rust/src/cmd/src/cli/tests.rs index f7f6d0a658..dac00ac983 100644 --- a/rust/src/cmd/src/cli/tests.rs +++ b/rust/src/cmd/src/cli/tests.rs @@ -200,6 +200,7 @@ fn serve_args_forward_python_flags_with_separator() { enable_prompt_tokens_details: false, enable_request_id_headers: false, enable_scale_out: false, + sse_keep_alive_interval: 0, disable_log_stats: false, served_model_name: [], allowed_origins: JsonStringList( @@ -1040,6 +1041,7 @@ fn frontend_args_accept_json() { enable_prompt_tokens_details: false, enable_request_id_headers: false, enable_scale_out: false, + sse_keep_alive_interval: 0, disable_log_stats: false, served_model_name: [], allowed_origins: JsonStringList( @@ -1711,6 +1713,7 @@ fn serve_args_accept_handshake_aliases() { enable_prompt_tokens_details: false, enable_request_id_headers: false, enable_scale_out: false, + sse_keep_alive_interval: 0, disable_log_stats: false, served_model_name: [], allowed_origins: JsonStringList( @@ -1867,6 +1870,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() { enable_prompt_tokens_details: false, enable_request_id_headers: false, enable_scale_out: false, + sse_keep_alive_interval: None, }, cors: CorsConfig { allow_origins: [ @@ -1960,6 +1964,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() { enable_prompt_tokens_details: false, enable_request_id_headers: false, enable_scale_out: false, + sse_keep_alive_interval: None, }, cors: CorsConfig { allow_origins: [ @@ -2074,6 +2079,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present enable_prompt_tokens_details: false, enable_request_id_headers: false, enable_scale_out: false, + sse_keep_alive_interval: None, }, cors: CorsConfig { allow_origins: [ diff --git a/rust/src/server/src/config.rs b/rust/src/server/src/config.rs index afcfaeccfb..c1c580e75a 100644 --- a/rust/src/server/src/config.rs +++ b/rust/src/server/src/config.rs @@ -59,6 +59,9 @@ pub struct ApiServerOptions { pub enable_request_id_headers: bool, /// When `true`, register the scale-out `/inference/v1/generate` route. pub enable_scale_out: bool, + /// Idle interval after which streaming SSE responses send a keep-alive + /// comment. `None` disables keep-alive comments. + pub sse_keep_alive_interval: Option, } /// CORS settings mirroring Python's `CORSMiddleware`; the default is permissive. diff --git a/rust/src/server/src/routes/inference/generate.rs b/rust/src/server/src/routes/inference/generate.rs index 3523f173e3..79b68622cf 100644 --- a/rust/src/server/src/routes/inference/generate.rs +++ b/rust/src/server/src/routes/inference/generate.rs @@ -14,7 +14,7 @@ use asynk_strim_attr::{TryYielder, try_stream}; use axum::Json; use axum::extract::State; use axum::http::HeaderMap; -use axum::response::sse::{Event, Sse}; +use axum::response::sse::Event; use axum::response::{IntoResponse, Response}; use futures::{Stream, StreamExt as _, pin_mut}; use thiserror_ext::AsReport as _; @@ -40,7 +40,7 @@ use crate::routes::openai::utils::logprobs::clamp_logprob; use crate::routes::openai::utils::types::{ChatLogProbs, ChatLogProbsContent, TopLogProb, Usage}; use crate::routes::openai::utils::validated_json::ValidatedJson; use crate::state::AppState; -use crate::utils::resolve_request_context; +use crate::utils::{resolve_request_context, sse_response}; /// Validate one token-in/token-out request and proxy it into the shared /// `vllm-text` stack. @@ -114,7 +114,7 @@ pub async fn generate( ); let sse_stream = generate_sse_stream(chunk_stream).instrument(request_span); - return Sse::new(sse_stream).into_response(); + return sse_response(sse_stream, api_server_options.sse_keep_alive_interval); } let collected = match raw_stream.collect_output().instrument(request_span.clone()).await { diff --git a/rust/src/server/src/routes/openai/chat_completions.rs b/rust/src/server/src/routes/openai/chat_completions.rs index 474719a875..a87c52fb55 100644 --- a/rust/src/server/src/routes/openai/chat_completions.rs +++ b/rust/src/server/src/routes/openai/chat_completions.rs @@ -13,7 +13,7 @@ use asynk_strim_attr::{TryYielder, try_stream}; use axum::Json; use axum::extract::State; use axum::http::HeaderMap; -use axum::response::sse::{Event, Sse}; +use axum::response::sse::Event; use axum::response::{IntoResponse, Response}; use futures::{Stream, StreamExt as _, pin_mut}; use serde_json::Value; @@ -45,7 +45,7 @@ use crate::routes::openai::utils::types::{ use crate::routes::openai::utils::usage::ContinuousUsage; use crate::routes::openai::utils::validated_json::ValidatedJson; use crate::state::AppState; -use crate::utils::{ResolvedRequestContext, resolve_request_context, unix_timestamp}; +use crate::utils::{ResolvedRequestContext, resolve_request_context, sse_response, unix_timestamp}; pub(crate) fn lower_chat_request( request: ChatCompletionRequest, @@ -99,7 +99,7 @@ pub async fn chat_completions( ); let sse_stream = chat_completion_sse_stream(chunk_stream).instrument(request_span); - Sse::new(sse_stream).into_response() + sse_response(sse_stream, api_server_options.sse_keep_alive_interval) } else { let response = match collect_chat_completion( chat_stream, diff --git a/rust/src/server/src/routes/openai/completions.rs b/rust/src/server/src/routes/openai/completions.rs index 06874a43bd..7074b8c629 100644 --- a/rust/src/server/src/routes/openai/completions.rs +++ b/rust/src/server/src/routes/openai/completions.rs @@ -13,7 +13,7 @@ use asynk_strim_attr::{TryYielder, try_stream}; use axum::Json; use axum::extract::State; use axum::http::HeaderMap; -use axum::response::sse::{Event, Sse}; +use axum::response::sse::Event; use axum::response::{IntoResponse, Response}; use futures::{Stream, StreamExt as _, pin_mut}; use serde_json::Value; @@ -45,7 +45,7 @@ use crate::routes::openai::utils::types::LogProbs; use crate::routes::openai::utils::usage::ContinuousUsage; use crate::routes::openai::utils::validated_json::ValidatedJson; use crate::state::AppState; -use crate::utils::{ResolvedRequestContext, resolve_request_context, unix_timestamp}; +use crate::utils::{ResolvedRequestContext, resolve_request_context, sse_response, unix_timestamp}; pub(crate) fn lower_completion_request( request: CompletionRequest, @@ -111,7 +111,7 @@ pub async fn completions( ); let sse_stream = completion_sse_stream(chunk_stream).instrument(request_span); - Sse::new(sse_stream).into_response() + sse_response(sse_stream, api_server_options.sse_keep_alive_interval) } else { let response = match collect_completion( text_stream, diff --git a/rust/src/server/src/utils.rs b/rust/src/server/src/utils.rs index b58e89fd3f..c3a8449617 100644 --- a/rust/src/server/src/utils.rs +++ b/rust/src/server/src/utils.rs @@ -2,9 +2,13 @@ // SPDX-FileCopyrightText: Copyright contributors to the vLLM project use std::collections::HashMap; -use std::time::{SystemTime, UNIX_EPOCH}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use axum::BoxError; use axum::http::HeaderMap; +use axum::response::sse::{Event, KeepAlive, Sse}; +use axum::response::{IntoResponse, Response}; +use futures::Stream; use serde_json::Value; use thiserror_ext::AsReport; use uuid::Uuid; @@ -27,6 +31,27 @@ pub fn unix_timestamp() -> u64 { .unwrap_or_default() } +/// Build a streaming SSE response, optionally sending keep-alive comments +/// while the stream is idle. +/// +/// With `keep_alive_interval` set, a `: keep-alive` comment is sent whenever no +/// event has been produced for that long (queued, prefill, between tokens, or +/// while a parser buffers a long tool call). SSE parsers ignore comments, but +/// they still count as bytes for read timeouts in reverse proxies and clients. +pub fn sse_response(stream: S, keep_alive_interval: Option) -> Response +where + S: Stream> + Send + 'static, + E: Into, +{ + let sse = Sse::new(stream); + match keep_alive_interval { + Some(interval) => sse + .keep_alive(KeepAlive::new().interval(interval).text("keep-alive")) + .into_response(), + None => sse.into_response(), + } +} + /// Construct an API error for a failed utility call to the engine core. pub fn utility_call_error(method: &str, error: impl AsReport) -> ApiError { ApiError::server_error(format!("failed to call {method}: {}", error.as_report())) @@ -157,3 +182,84 @@ pub fn resolve_base_request_id( id }) } + +#[cfg(test)] +mod tests { + use std::convert::Infallible; + + use futures::{StreamExt as _, stream}; + use tokio::time::{Instant, sleep}; + + use super::*; + + /// Collect `(elapsed seconds, frame)` pairs from a response whose event + /// stream idles for 20 seconds between its two events. + async fn idle_stream_frames(keep_alive_interval: Option) -> Vec<(u64, String)> { + let events = stream::iter([Ok::<_, Infallible>(Event::default().data("first"))]).chain( + stream::once(async { + sleep(Duration::from_secs(20)).await; + Ok(Event::default().data("second")) + }), + ); + let start = Instant::now(); + sse_response(events, keep_alive_interval) + .into_body() + .into_data_stream() + .map(|frame| { + let frame = frame.expect("read frame"); + ( + start.elapsed().as_secs(), + String::from_utf8(frame.to_vec()).expect("utf-8 frame"), + ) + }) + .collect() + .await + } + + /// An idle gap is filled with `: keep-alive` comments at the configured + /// interval, and real events still pass through in order. + #[tokio::test(start_paused = true)] + async fn sse_response_sends_keep_alive_comments_while_idle() { + let frames = idle_stream_frames(Some(Duration::from_secs(8))).await; + expect_test::expect![[r#" + [ + ( + 0, + "data: first\n\n", + ), + ( + 8, + ": keep-alive\n\n", + ), + ( + 16, + ": keep-alive\n\n", + ), + ( + 20, + "data: second\n\n", + ), + ] + "#]] + .assert_debug_eq(&frames); + } + + /// Without an interval, an idle gap produces no extra bytes. + #[tokio::test(start_paused = true)] + async fn sse_response_without_keep_alive_interval_sends_no_comments() { + let frames = idle_stream_frames(None).await; + expect_test::expect![[r#" + [ + ( + 0, + "data: first\n\n", + ), + ( + 20, + "data: second\n\n", + ), + ] + "#]] + .assert_debug_eq(&frames); + } +}