[Rust Frontend] Support --sse-keep-alive-interval (#58306)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
cf78797cb7
commit
e028554a1f
@@ -363,6 +363,14 @@ pub struct SharedRuntimeArgs {
|
||||
#[serde(default)]
|
||||
pub enable_scale_out: bool,
|
||||
|
||||
/// Send an SSE keep-alive comment line every this many seconds when a
|
||||
/// streaming response is idle (queued, prefill, or between tokens), to
|
||||
/// prevent reverse proxies/tunnels with read timeouts from closing the
|
||||
/// connection. Defaults to 0, which disables keep-alive comments entirely.
|
||||
#[arg(long, default_value_t = 0)]
|
||||
#[serde(default)]
|
||||
pub sse_keep_alive_interval: u64,
|
||||
|
||||
/// If provided, the server will require one of these keys to be presented
|
||||
/// in the Authorization header.
|
||||
#[educe(Debug(ignore))]
|
||||
@@ -616,6 +624,8 @@ impl SharedRuntimeArgs {
|
||||
enable_prompt_tokens_details: self.enable_prompt_tokens_details,
|
||||
enable_request_id_headers: self.enable_request_id_headers,
|
||||
enable_scale_out: self.enable_scale_out,
|
||||
sse_keep_alive_interval: (self.sse_keep_alive_interval > 0)
|
||||
.then(|| Duration::from_secs(self.sse_keep_alive_interval)),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -200,6 +200,7 @@ fn serve_args_forward_python_flags_with_separator() {
|
||||
enable_prompt_tokens_details: false,
|
||||
enable_request_id_headers: false,
|
||||
enable_scale_out: false,
|
||||
sse_keep_alive_interval: 0,
|
||||
disable_log_stats: false,
|
||||
served_model_name: [],
|
||||
allowed_origins: JsonStringList(
|
||||
@@ -1040,6 +1041,7 @@ fn frontend_args_accept_json() {
|
||||
enable_prompt_tokens_details: false,
|
||||
enable_request_id_headers: false,
|
||||
enable_scale_out: false,
|
||||
sse_keep_alive_interval: 0,
|
||||
disable_log_stats: false,
|
||||
served_model_name: [],
|
||||
allowed_origins: JsonStringList(
|
||||
@@ -1711,6 +1713,7 @@ fn serve_args_accept_handshake_aliases() {
|
||||
enable_prompt_tokens_details: false,
|
||||
enable_request_id_headers: false,
|
||||
enable_scale_out: false,
|
||||
sse_keep_alive_interval: 0,
|
||||
disable_log_stats: false,
|
||||
served_model_name: [],
|
||||
allowed_origins: JsonStringList(
|
||||
@@ -1867,6 +1870,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
|
||||
enable_prompt_tokens_details: false,
|
||||
enable_request_id_headers: false,
|
||||
enable_scale_out: false,
|
||||
sse_keep_alive_interval: None,
|
||||
},
|
||||
cors: CorsConfig {
|
||||
allow_origins: [
|
||||
@@ -1960,6 +1964,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
|
||||
enable_prompt_tokens_details: false,
|
||||
enable_request_id_headers: false,
|
||||
enable_scale_out: false,
|
||||
sse_keep_alive_interval: None,
|
||||
},
|
||||
cors: CorsConfig {
|
||||
allow_origins: [
|
||||
@@ -2074,6 +2079,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
|
||||
enable_prompt_tokens_details: false,
|
||||
enable_request_id_headers: false,
|
||||
enable_scale_out: false,
|
||||
sse_keep_alive_interval: None,
|
||||
},
|
||||
cors: CorsConfig {
|
||||
allow_origins: [
|
||||
|
||||
@@ -59,6 +59,9 @@ pub struct ApiServerOptions {
|
||||
pub enable_request_id_headers: bool,
|
||||
/// When `true`, register the scale-out `/inference/v1/generate` route.
|
||||
pub enable_scale_out: bool,
|
||||
/// Idle interval after which streaming SSE responses send a keep-alive
|
||||
/// comment. `None` disables keep-alive comments.
|
||||
pub sse_keep_alive_interval: Option<Duration>,
|
||||
}
|
||||
|
||||
/// CORS settings mirroring Python's `CORSMiddleware`; the default is permissive.
|
||||
|
||||
@@ -14,7 +14,7 @@ use asynk_strim_attr::{TryYielder, try_stream};
|
||||
use axum::Json;
|
||||
use axum::extract::State;
|
||||
use axum::http::HeaderMap;
|
||||
use axum::response::sse::{Event, Sse};
|
||||
use axum::response::sse::Event;
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use futures::{Stream, StreamExt as _, pin_mut};
|
||||
use thiserror_ext::AsReport as _;
|
||||
@@ -40,7 +40,7 @@ use crate::routes::openai::utils::logprobs::clamp_logprob;
|
||||
use crate::routes::openai::utils::types::{ChatLogProbs, ChatLogProbsContent, TopLogProb, Usage};
|
||||
use crate::routes::openai::utils::validated_json::ValidatedJson;
|
||||
use crate::state::AppState;
|
||||
use crate::utils::resolve_request_context;
|
||||
use crate::utils::{resolve_request_context, sse_response};
|
||||
|
||||
/// Validate one token-in/token-out request and proxy it into the shared
|
||||
/// `vllm-text` stack.
|
||||
@@ -114,7 +114,7 @@ pub async fn generate(
|
||||
);
|
||||
let sse_stream = generate_sse_stream(chunk_stream).instrument(request_span);
|
||||
|
||||
return Sse::new(sse_stream).into_response();
|
||||
return sse_response(sse_stream, api_server_options.sse_keep_alive_interval);
|
||||
}
|
||||
|
||||
let collected = match raw_stream.collect_output().instrument(request_span.clone()).await {
|
||||
|
||||
@@ -13,7 +13,7 @@ use asynk_strim_attr::{TryYielder, try_stream};
|
||||
use axum::Json;
|
||||
use axum::extract::State;
|
||||
use axum::http::HeaderMap;
|
||||
use axum::response::sse::{Event, Sse};
|
||||
use axum::response::sse::Event;
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use futures::{Stream, StreamExt as _, pin_mut};
|
||||
use serde_json::Value;
|
||||
@@ -45,7 +45,7 @@ use crate::routes::openai::utils::types::{
|
||||
use crate::routes::openai::utils::usage::ContinuousUsage;
|
||||
use crate::routes::openai::utils::validated_json::ValidatedJson;
|
||||
use crate::state::AppState;
|
||||
use crate::utils::{ResolvedRequestContext, resolve_request_context, unix_timestamp};
|
||||
use crate::utils::{ResolvedRequestContext, resolve_request_context, sse_response, unix_timestamp};
|
||||
|
||||
pub(crate) fn lower_chat_request(
|
||||
request: ChatCompletionRequest,
|
||||
@@ -99,7 +99,7 @@ pub async fn chat_completions(
|
||||
);
|
||||
let sse_stream = chat_completion_sse_stream(chunk_stream).instrument(request_span);
|
||||
|
||||
Sse::new(sse_stream).into_response()
|
||||
sse_response(sse_stream, api_server_options.sse_keep_alive_interval)
|
||||
} else {
|
||||
let response = match collect_chat_completion(
|
||||
chat_stream,
|
||||
|
||||
@@ -13,7 +13,7 @@ use asynk_strim_attr::{TryYielder, try_stream};
|
||||
use axum::Json;
|
||||
use axum::extract::State;
|
||||
use axum::http::HeaderMap;
|
||||
use axum::response::sse::{Event, Sse};
|
||||
use axum::response::sse::Event;
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use futures::{Stream, StreamExt as _, pin_mut};
|
||||
use serde_json::Value;
|
||||
@@ -45,7 +45,7 @@ use crate::routes::openai::utils::types::LogProbs;
|
||||
use crate::routes::openai::utils::usage::ContinuousUsage;
|
||||
use crate::routes::openai::utils::validated_json::ValidatedJson;
|
||||
use crate::state::AppState;
|
||||
use crate::utils::{ResolvedRequestContext, resolve_request_context, unix_timestamp};
|
||||
use crate::utils::{ResolvedRequestContext, resolve_request_context, sse_response, unix_timestamp};
|
||||
|
||||
pub(crate) fn lower_completion_request(
|
||||
request: CompletionRequest,
|
||||
@@ -111,7 +111,7 @@ pub async fn completions(
|
||||
);
|
||||
let sse_stream = completion_sse_stream(chunk_stream).instrument(request_span);
|
||||
|
||||
Sse::new(sse_stream).into_response()
|
||||
sse_response(sse_stream, api_server_options.sse_keep_alive_interval)
|
||||
} else {
|
||||
let response = match collect_completion(
|
||||
text_stream,
|
||||
|
||||
@@ -2,9 +2,13 @@
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use axum::BoxError;
|
||||
use axum::http::HeaderMap;
|
||||
use axum::response::sse::{Event, KeepAlive, Sse};
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use futures::Stream;
|
||||
use serde_json::Value;
|
||||
use thiserror_ext::AsReport;
|
||||
use uuid::Uuid;
|
||||
@@ -27,6 +31,27 @@ pub fn unix_timestamp() -> u64 {
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Build a streaming SSE response, optionally sending keep-alive comments
|
||||
/// while the stream is idle.
|
||||
///
|
||||
/// With `keep_alive_interval` set, a `: keep-alive` comment is sent whenever no
|
||||
/// event has been produced for that long (queued, prefill, between tokens, or
|
||||
/// while a parser buffers a long tool call). SSE parsers ignore comments, but
|
||||
/// they still count as bytes for read timeouts in reverse proxies and clients.
|
||||
pub fn sse_response<S, E>(stream: S, keep_alive_interval: Option<Duration>) -> Response
|
||||
where
|
||||
S: Stream<Item = Result<Event, E>> + Send + 'static,
|
||||
E: Into<BoxError>,
|
||||
{
|
||||
let sse = Sse::new(stream);
|
||||
match keep_alive_interval {
|
||||
Some(interval) => sse
|
||||
.keep_alive(KeepAlive::new().interval(interval).text("keep-alive"))
|
||||
.into_response(),
|
||||
None => sse.into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Construct an API error for a failed utility call to the engine core.
|
||||
pub fn utility_call_error(method: &str, error: impl AsReport) -> ApiError {
|
||||
ApiError::server_error(format!("failed to call {method}: {}", error.as_report()))
|
||||
@@ -157,3 +182,84 @@ pub fn resolve_base_request_id(
|
||||
id
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::convert::Infallible;
|
||||
|
||||
use futures::{StreamExt as _, stream};
|
||||
use tokio::time::{Instant, sleep};
|
||||
|
||||
use super::*;
|
||||
|
||||
/// Collect `(elapsed seconds, frame)` pairs from a response whose event
|
||||
/// stream idles for 20 seconds between its two events.
|
||||
async fn idle_stream_frames(keep_alive_interval: Option<Duration>) -> Vec<(u64, String)> {
|
||||
let events = stream::iter([Ok::<_, Infallible>(Event::default().data("first"))]).chain(
|
||||
stream::once(async {
|
||||
sleep(Duration::from_secs(20)).await;
|
||||
Ok(Event::default().data("second"))
|
||||
}),
|
||||
);
|
||||
let start = Instant::now();
|
||||
sse_response(events, keep_alive_interval)
|
||||
.into_body()
|
||||
.into_data_stream()
|
||||
.map(|frame| {
|
||||
let frame = frame.expect("read frame");
|
||||
(
|
||||
start.elapsed().as_secs(),
|
||||
String::from_utf8(frame.to_vec()).expect("utf-8 frame"),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
.await
|
||||
}
|
||||
|
||||
/// An idle gap is filled with `: keep-alive` comments at the configured
|
||||
/// interval, and real events still pass through in order.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn sse_response_sends_keep_alive_comments_while_idle() {
|
||||
let frames = idle_stream_frames(Some(Duration::from_secs(8))).await;
|
||||
expect_test::expect![[r#"
|
||||
[
|
||||
(
|
||||
0,
|
||||
"data: first\n\n",
|
||||
),
|
||||
(
|
||||
8,
|
||||
": keep-alive\n\n",
|
||||
),
|
||||
(
|
||||
16,
|
||||
": keep-alive\n\n",
|
||||
),
|
||||
(
|
||||
20,
|
||||
"data: second\n\n",
|
||||
),
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&frames);
|
||||
}
|
||||
|
||||
/// Without an interval, an idle gap produces no extra bytes.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn sse_response_without_keep_alive_interval_sends_no_comments() {
|
||||
let frames = idle_stream_frames(None).await;
|
||||
expect_test::expect![[r#"
|
||||
[
|
||||
(
|
||||
0,
|
||||
"data: first\n\n",
|
||||
),
|
||||
(
|
||||
20,
|
||||
"data: second\n\n",
|
||||
),
|
||||
]
|
||||
"#]]
|
||||
.assert_debug_eq(&frames);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user