[Rust Frontend] Support --stream-interval and per-request stream_interval (#60589)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Bugen Zhao
2026-10-09 14:10:42 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent ffdde24f54
commit 35e920d671
25 changed files with 1040 additions and 154 deletions
+14
View File
@@ -12,6 +12,7 @@ mod unsupported;
use std::collections::HashMap;
use std::ffi::{OsStr, OsString};
use std::num::NonZeroU32;
use std::path::PathBuf;
use std::time::Duration;
@@ -268,6 +269,13 @@ pub struct SharedRuntimeArgs {
#[arg(long, value_parser = clap::value_parser!(i32).range(-1..), allow_negative_numbers = true)]
#[serde(default)]
pub max_logprobs: Option<i32>,
/// The interval (or buffer size) for streaming in terms of token length.
/// A smaller value (1) makes streaming smoother by sending each token
/// immediately, while a larger value (e.g., 10) reduces host overhead and
/// may increase throughput by batching multiple tokens before sending.
#[arg(long, default_value_t = NonZeroU32::MIN)]
#[serde(default = "default_stream_interval")]
pub stream_interval: NonZeroU32,
/// Maximum time to wait for active requests to drain during shutdown.
#[arg(long, default_value_t = 0)]
#[serde(default)]
@@ -547,6 +555,7 @@ impl SharedRuntimeArgs {
lora_modules: self.lora_modules,
chat_template_content_format: self.chat_template_content_format,
max_logprobs: self.max_logprobs,
stream_interval: self.stream_interval,
api_server_options,
cors,
tls,
@@ -609,6 +618,7 @@ impl SharedRuntimeArgs {
lora_modules: self.lora_modules,
chat_template_content_format: self.chat_template_content_format,
max_logprobs: self.max_logprobs,
stream_interval: self.stream_interval,
api_server_options,
cors,
tls,
@@ -646,6 +656,10 @@ fn default_engine_ready_timeout_secs() -> u64 {
600
}
fn default_stream_interval() -> NonZeroU32 {
NonZeroU32::MIN
}
fn default_cors_wildcard() -> JsonStringList {
JsonStringList(vec!["*".to_string()])
}
+34
View File
@@ -189,6 +189,7 @@ fn serve_args_forward_python_flags_with_separator() {
renderer: Auto,
language_model_only: false,
max_logprobs: None,
stream_interval: 1,
shutdown_timeout: 0,
http_timeout_keep_alive: None,
chat_template: None,
@@ -373,6 +374,34 @@ fn serve_args_forward_profiler_config_to_managed_engine() {
);
}
#[test]
fn serve_args_forward_stream_interval_to_frontend_config() {
let cli = Cli::try_parse_from([
"vllm-rs",
"serve",
"Qwen/Qwen3-0.6B",
"--stream-interval",
"4",
])
.unwrap();
let Command::Serve(args) = cli.command else {
panic!("expected serve args");
};
let frontend_config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
assert_eq!(frontend_config.stream_interval.get(), 4);
let error = Cli::try_parse_from([
"vllm-rs",
"serve",
"Qwen/Qwen3-0.6B",
"--stream-interval",
"0",
])
.unwrap_err();
assert_eq!(error.kind(), clap::error::ErrorKind::ValueValidation);
}
#[test]
fn serve_args_forward_max_logprobs_to_frontend_and_managed_engine() {
let cli = Cli::try_parse_from([
@@ -1030,6 +1059,7 @@ fn frontend_args_accept_json() {
renderer: Auto,
language_model_only: false,
max_logprobs: None,
stream_interval: 1,
shutdown_timeout: 0,
http_timeout_keep_alive: None,
chat_template: None,
@@ -1702,6 +1732,7 @@ fn serve_args_accept_handshake_aliases() {
renderer: Auto,
language_model_only: false,
max_logprobs: None,
stream_interval: 1,
shutdown_timeout: 0,
http_timeout_keep_alive: None,
chat_template: None,
@@ -1866,6 +1897,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
lora_modules: [],
chat_template_content_format: Auto,
max_logprobs: None,
stream_interval: 1,
api_server_options: ApiServerOptions {
enable_log_requests: false,
enable_prompt_tokens_details: false,
@@ -1961,6 +1993,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
lora_modules: [],
chat_template_content_format: Auto,
max_logprobs: None,
stream_interval: 1,
api_server_options: ApiServerOptions {
enable_log_requests: false,
enable_prompt_tokens_details: false,
@@ -2100,6 +2133,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
lora_modules: [],
chat_template_content_format: Auto,
max_logprobs: None,
stream_interval: 1,
api_server_options: ApiServerOptions {
enable_log_requests: false,
enable_prompt_tokens_details: false,
-4
View File
@@ -359,10 +359,6 @@ pub struct EngineUnsupportedArgs {
#[arg(long)]
pub collect_detailed_traces: Option<Unsupported>,
/// The interval (or buffer size) for streaming in terms of token length.
#[arg(long)]
pub stream_interval: Option<Unsupported>,
/// Maximum number of requests that can be in-flight (waiting or running)
/// at the same time. When the limit is reached, new requests are rejected
/// with HTTP 503 so the client can retry on another instance.