[Rust Frontend] Support --sse-keep-alive-interval (#58306)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-09-24 06:19:14 +00:00
committed by GitHub
co-authored by Claude Opus 5.5 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent cf78797cb7
commit e028554a1f
7 changed files with 135 additions and 10 deletions
+10
View File
@@ -363,6 +363,14 @@ pub struct SharedRuntimeArgs {
#[serde(default)]
pub enable_scale_out: bool,
/// Send an SSE keep-alive comment line every this many seconds when a
/// streaming response is idle (queued, prefill, or between tokens), to
/// prevent reverse proxies/tunnels with read timeouts from closing the
/// connection. Defaults to 0, which disables keep-alive comments entirely.
#[arg(long, default_value_t = 0)]
#[serde(default)]
pub sse_keep_alive_interval: u64,
/// If provided, the server will require one of these keys to be presented
/// in the Authorization header.
#[educe(Debug(ignore))]
@@ -616,6 +624,8 @@ impl SharedRuntimeArgs {
enable_prompt_tokens_details: self.enable_prompt_tokens_details,
enable_request_id_headers: self.enable_request_id_headers,
enable_scale_out: self.enable_scale_out,
sse_keep_alive_interval: (self.sse_keep_alive_interval > 0)
.then(|| Duration::from_secs(self.sse_keep_alive_interval)),
}
}
+6
View File
@@ -200,6 +200,7 @@ fn serve_args_forward_python_flags_with_separator() {
enable_prompt_tokens_details: false,
enable_request_id_headers: false,
enable_scale_out: false,
sse_keep_alive_interval: 0,
disable_log_stats: false,
served_model_name: [],
allowed_origins: JsonStringList(
@@ -1040,6 +1041,7 @@ fn frontend_args_accept_json() {
enable_prompt_tokens_details: false,
enable_request_id_headers: false,
enable_scale_out: false,
sse_keep_alive_interval: 0,
disable_log_stats: false,
served_model_name: [],
allowed_origins: JsonStringList(
@@ -1711,6 +1713,7 @@ fn serve_args_accept_handshake_aliases() {
enable_prompt_tokens_details: false,
enable_request_id_headers: false,
enable_scale_out: false,
sse_keep_alive_interval: 0,
disable_log_stats: false,
served_model_name: [],
allowed_origins: JsonStringList(
@@ -1867,6 +1870,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
enable_prompt_tokens_details: false,
enable_request_id_headers: false,
enable_scale_out: false,
sse_keep_alive_interval: None,
},
cors: CorsConfig {
allow_origins: [
@@ -1960,6 +1964,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
enable_prompt_tokens_details: false,
enable_request_id_headers: false,
enable_scale_out: false,
sse_keep_alive_interval: None,
},
cors: CorsConfig {
allow_origins: [
@@ -2074,6 +2079,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
enable_prompt_tokens_details: false,
enable_request_id_headers: false,
enable_scale_out: false,
sse_keep_alive_interval: None,
},
cors: CorsConfig {
allow_origins: [