[Rust Frontend] Support --stream-interval and per-request stream_interval (#60589)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
ffdde24f54
commit
35e920d671
@@ -12,6 +12,7 @@ mod unsupported;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::ffi::{OsStr, OsString};
|
||||
use std::num::NonZeroU32;
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -268,6 +269,13 @@ pub struct SharedRuntimeArgs {
|
||||
#[arg(long, value_parser = clap::value_parser!(i32).range(-1..), allow_negative_numbers = true)]
|
||||
#[serde(default)]
|
||||
pub max_logprobs: Option<i32>,
|
||||
/// The interval (or buffer size) for streaming in terms of token length.
|
||||
/// A smaller value (1) makes streaming smoother by sending each token
|
||||
/// immediately, while a larger value (e.g., 10) reduces host overhead and
|
||||
/// may increase throughput by batching multiple tokens before sending.
|
||||
#[arg(long, default_value_t = NonZeroU32::MIN)]
|
||||
#[serde(default = "default_stream_interval")]
|
||||
pub stream_interval: NonZeroU32,
|
||||
/// Maximum time to wait for active requests to drain during shutdown.
|
||||
#[arg(long, default_value_t = 0)]
|
||||
#[serde(default)]
|
||||
@@ -547,6 +555,7 @@ impl SharedRuntimeArgs {
|
||||
lora_modules: self.lora_modules,
|
||||
chat_template_content_format: self.chat_template_content_format,
|
||||
max_logprobs: self.max_logprobs,
|
||||
stream_interval: self.stream_interval,
|
||||
api_server_options,
|
||||
cors,
|
||||
tls,
|
||||
@@ -609,6 +618,7 @@ impl SharedRuntimeArgs {
|
||||
lora_modules: self.lora_modules,
|
||||
chat_template_content_format: self.chat_template_content_format,
|
||||
max_logprobs: self.max_logprobs,
|
||||
stream_interval: self.stream_interval,
|
||||
api_server_options,
|
||||
cors,
|
||||
tls,
|
||||
@@ -646,6 +656,10 @@ fn default_engine_ready_timeout_secs() -> u64 {
|
||||
600
|
||||
}
|
||||
|
||||
fn default_stream_interval() -> NonZeroU32 {
|
||||
NonZeroU32::MIN
|
||||
}
|
||||
|
||||
fn default_cors_wildcard() -> JsonStringList {
|
||||
JsonStringList(vec!["*".to_string()])
|
||||
}
|
||||
|
||||
@@ -189,6 +189,7 @@ fn serve_args_forward_python_flags_with_separator() {
|
||||
renderer: Auto,
|
||||
language_model_only: false,
|
||||
max_logprobs: None,
|
||||
stream_interval: 1,
|
||||
shutdown_timeout: 0,
|
||||
http_timeout_keep_alive: None,
|
||||
chat_template: None,
|
||||
@@ -373,6 +374,34 @@ fn serve_args_forward_profiler_config_to_managed_engine() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_forward_stream_interval_to_frontend_config() {
|
||||
let cli = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--stream-interval",
|
||||
"4",
|
||||
])
|
||||
.unwrap();
|
||||
|
||||
let Command::Serve(args) = cli.command else {
|
||||
panic!("expected serve args");
|
||||
};
|
||||
let frontend_config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
|
||||
assert_eq!(frontend_config.stream_interval.get(), 4);
|
||||
|
||||
let error = Cli::try_parse_from([
|
||||
"vllm-rs",
|
||||
"serve",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"--stream-interval",
|
||||
"0",
|
||||
])
|
||||
.unwrap_err();
|
||||
assert_eq!(error.kind(), clap::error::ErrorKind::ValueValidation);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_args_forward_max_logprobs_to_frontend_and_managed_engine() {
|
||||
let cli = Cli::try_parse_from([
|
||||
@@ -1030,6 +1059,7 @@ fn frontend_args_accept_json() {
|
||||
renderer: Auto,
|
||||
language_model_only: false,
|
||||
max_logprobs: None,
|
||||
stream_interval: 1,
|
||||
shutdown_timeout: 0,
|
||||
http_timeout_keep_alive: None,
|
||||
chat_template: None,
|
||||
@@ -1702,6 +1732,7 @@ fn serve_args_accept_handshake_aliases() {
|
||||
renderer: Auto,
|
||||
language_model_only: false,
|
||||
max_logprobs: None,
|
||||
stream_interval: 1,
|
||||
shutdown_timeout: 0,
|
||||
http_timeout_keep_alive: None,
|
||||
chat_template: None,
|
||||
@@ -1866,6 +1897,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
|
||||
lora_modules: [],
|
||||
chat_template_content_format: Auto,
|
||||
max_logprobs: None,
|
||||
stream_interval: 1,
|
||||
api_server_options: ApiServerOptions {
|
||||
enable_log_requests: false,
|
||||
enable_prompt_tokens_details: false,
|
||||
@@ -1961,6 +1993,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
|
||||
lora_modules: [],
|
||||
chat_template_content_format: Auto,
|
||||
max_logprobs: None,
|
||||
stream_interval: 1,
|
||||
api_server_options: ApiServerOptions {
|
||||
enable_log_requests: false,
|
||||
enable_prompt_tokens_details: false,
|
||||
@@ -2100,6 +2133,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
|
||||
lora_modules: [],
|
||||
chat_template_content_format: Auto,
|
||||
max_logprobs: None,
|
||||
stream_interval: 1,
|
||||
api_server_options: ApiServerOptions {
|
||||
enable_log_requests: false,
|
||||
enable_prompt_tokens_details: false,
|
||||
|
||||
@@ -359,10 +359,6 @@ pub struct EngineUnsupportedArgs {
|
||||
#[arg(long)]
|
||||
pub collect_detailed_traces: Option<Unsupported>,
|
||||
|
||||
/// The interval (or buffer size) for streaming in terms of token length.
|
||||
#[arg(long)]
|
||||
pub stream_interval: Option<Unsupported>,
|
||||
|
||||
/// Maximum number of requests that can be in-flight (waiting or running)
|
||||
/// at the same time. When the limit is reached, new requests are rejected
|
||||
/// with HTTP 503 so the client can retry on another instance.
|
||||
|
||||
Reference in New Issue
Block a user