[Rust Frontend] Support --sse-keep-alive-interval (#58306)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
cf78797cb7
commit
e028554a1f
@@ -363,6 +363,14 @@ pub struct SharedRuntimeArgs {
|
||||
#[serde(default)]
|
||||
pub enable_scale_out: bool,
|
||||
|
||||
/// Send an SSE keep-alive comment line every this many seconds when a
|
||||
/// streaming response is idle (queued, prefill, or between tokens), to
|
||||
/// prevent reverse proxies/tunnels with read timeouts from closing the
|
||||
/// connection. Defaults to 0, which disables keep-alive comments entirely.
|
||||
#[arg(long, default_value_t = 0)]
|
||||
#[serde(default)]
|
||||
pub sse_keep_alive_interval: u64,
|
||||
|
||||
/// If provided, the server will require one of these keys to be presented
|
||||
/// in the Authorization header.
|
||||
#[educe(Debug(ignore))]
|
||||
@@ -616,6 +624,8 @@ impl SharedRuntimeArgs {
|
||||
enable_prompt_tokens_details: self.enable_prompt_tokens_details,
|
||||
enable_request_id_headers: self.enable_request_id_headers,
|
||||
enable_scale_out: self.enable_scale_out,
|
||||
sse_keep_alive_interval: (self.sse_keep_alive_interval > 0)
|
||||
.then(|| Duration::from_secs(self.sse_keep_alive_interval)),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -200,6 +200,7 @@ fn serve_args_forward_python_flags_with_separator() {
|
||||
enable_prompt_tokens_details: false,
|
||||
enable_request_id_headers: false,
|
||||
enable_scale_out: false,
|
||||
sse_keep_alive_interval: 0,
|
||||
disable_log_stats: false,
|
||||
served_model_name: [],
|
||||
allowed_origins: JsonStringList(
|
||||
@@ -1040,6 +1041,7 @@ fn frontend_args_accept_json() {
|
||||
enable_prompt_tokens_details: false,
|
||||
enable_request_id_headers: false,
|
||||
enable_scale_out: false,
|
||||
sse_keep_alive_interval: 0,
|
||||
disable_log_stats: false,
|
||||
served_model_name: [],
|
||||
allowed_origins: JsonStringList(
|
||||
@@ -1711,6 +1713,7 @@ fn serve_args_accept_handshake_aliases() {
|
||||
enable_prompt_tokens_details: false,
|
||||
enable_request_id_headers: false,
|
||||
enable_scale_out: false,
|
||||
sse_keep_alive_interval: 0,
|
||||
disable_log_stats: false,
|
||||
served_model_name: [],
|
||||
allowed_origins: JsonStringList(
|
||||
@@ -1867,6 +1870,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
|
||||
enable_prompt_tokens_details: false,
|
||||
enable_request_id_headers: false,
|
||||
enable_scale_out: false,
|
||||
sse_keep_alive_interval: None,
|
||||
},
|
||||
cors: CorsConfig {
|
||||
allow_origins: [
|
||||
@@ -1960,6 +1964,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
|
||||
enable_prompt_tokens_details: false,
|
||||
enable_request_id_headers: false,
|
||||
enable_scale_out: false,
|
||||
sse_keep_alive_interval: None,
|
||||
},
|
||||
cors: CorsConfig {
|
||||
allow_origins: [
|
||||
@@ -2074,6 +2079,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
|
||||
enable_prompt_tokens_details: false,
|
||||
enable_request_id_headers: false,
|
||||
enable_scale_out: false,
|
||||
sse_keep_alive_interval: None,
|
||||
},
|
||||
cors: CorsConfig {
|
||||
allow_origins: [
|
||||
|
||||
Reference in New Issue
Block a user