[Rust Frontend] Support --stream-interval and per-request stream_interval (#60589)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Bugen Zhao
2026-10-09 14:10:42 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent ffdde24f54
commit 35e920d671
25 changed files with 1040 additions and 154 deletions
@@ -2,6 +2,7 @@
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::collections::HashMap;
use std::num::NonZeroU32;
use std::time::Duration;
use anyhow::{Context, Result, bail};
@@ -81,6 +82,7 @@ async fn main() -> Result<()> {
lora_modules: Vec::new(),
chat_template_content_format: ChatTemplateContentFormatOption::Auto,
max_logprobs: None,
stream_interval: NonZeroU32::MIN,
api_server_options: ApiServerOptions::default(),
cors: CorsConfig::default(),
tls: None,
+5
View File
@@ -3,6 +3,7 @@
use std::collections::HashMap;
use std::fmt;
use std::num::NonZeroU32;
use std::str::FromStr;
use std::time::Duration;
@@ -258,6 +259,10 @@ pub struct Config {
/// Optional maximum number of top log probabilities accepted by the
/// frontend. `None` delegates to the text layer default.
pub max_logprobs: Option<i32>,
/// Minimum number of newly generated tokens batched into each streamed
/// output after the first one. Requests can raise it with their own
/// `stream_interval`.
pub stream_interval: NonZeroU32,
/// HTTP/API-server behavior switches.
pub api_server_options: ApiServerOptions,
/// CORS settings applied to every HTTP response.
+3 -1
View File
@@ -123,7 +123,9 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
.await
.context("failed to connect to engine core")?;
let llm = Llm::new(client).with_log_stats(!config.disable_log_stats);
let llm = Llm::new(client)
.with_log_stats(!config.disable_log_stats)
.with_stream_interval(config.stream_interval);
let text = TextLlm::new(llm, text_backend).with_max_logprobs(config.max_logprobs);
let chat = ChatLlm::new(text, chat_backend)
@@ -179,6 +179,7 @@ pub(super) fn prepare_chat_request(
merge_ec_transfer_params(request.vllm_xargs, request.ec_transfer_params.as_ref()),
request.kv_transfer_params.as_ref(),
),
stream_interval: request.stream_interval,
},
chat_options: ChatOptions {
generation_prompt_mode,
@@ -443,6 +444,7 @@ fn convert_tool_choice(tool_choice: &ToolChoice) -> Result<ChatToolChoice, ApiEr
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use std::num::NonZeroU32;
use std::sync::Arc;
use axum::http::{HeaderMap, StatusCode};
@@ -540,6 +542,15 @@ mod tests {
}
}
#[test]
fn chat_http_request_rejects_zero_stream_interval() {
let request = json!({
"messages": [{"role": "user", "content": "hello"}],
"stream_interval": 0,
});
assert!(serde_json::from_value::<ChatCompletionRequest>(request).is_err());
}
#[test]
fn chat_http_request_defaults_missing_or_null_model() {
for model in [None, Some(serde_json::Value::Null)] {
@@ -1004,6 +1015,7 @@ mod tests {
frequency_penalty: Some(0.3),
presence_penalty: Some(0.4),
repetition_penalty: Some(1.1),
stream_interval: NonZeroU32::new(4),
..base_request()
};
@@ -1019,6 +1031,7 @@ mod tests {
frequency_penalty: Some(0.3),
presence_penalty: Some(0.4),
repetition_penalty: Some(1.1),
stream_interval: NonZeroU32::new(4),
..VllmSamplingParams::default()
};
assert_eq!(prepared.chat_request.sampling_params, expected);
@@ -3,6 +3,7 @@
use std::collections::HashMap;
use std::fmt;
use std::num::NonZeroU32;
use std::sync::Arc;
use serde::{Deserialize, Serialize};
@@ -256,6 +257,12 @@ pub struct ChatCompletionRequest {
/// Parameters for detecting repetitive N-gram patterns in output tokens
pub repetition_detection: Option<RepetitionDetectionParams>,
/// Number of tokens to batch into each streamed chunk. Raises the server's
/// `--stream-interval` for this request. Values below the server setting
/// are clamped up to it. The first and last chunks are always sent
/// immediately. Ignored for non-streaming requests.
pub stream_interval: Option<NonZeroU32>,
}
impl Default for ChatCompletionRequest {
@@ -323,6 +330,7 @@ impl Default for ChatCompletionRequest {
ec_transfer_params: None,
vllm_xargs: None,
repetition_detection: None,
stream_interval: None,
}
}
}
@@ -144,6 +144,7 @@ pub(super) fn prepare_completion_request(
merge_ec_transfer_params(request.vllm_xargs, request.ec_transfer_params.as_ref()),
request.kv_transfer_params.as_ref(),
),
stream_interval: request.stream_interval,
},
decode_options: TextDecodeOptions {
skip_special_tokens: request.skip_special_tokens,
@@ -211,6 +212,8 @@ fn completion_echo_text(
#[cfg(test)]
mod tests {
use std::num::NonZeroU32;
use axum::http::HeaderMap;
use serde_json::json;
use validator::Validate;
@@ -400,7 +403,8 @@ mod tests {
"presence_penalty": 0.3,
"repetition_penalty": 1.1,
"ignore_eos": true,
"skip_special_tokens": false
"skip_special_tokens": false,
"stream_interval": 4
}))
.expect("parse request");
@@ -436,6 +440,10 @@ mod tests {
);
assert!(prepared.text_request.sampling_params.ignore_eos);
assert!(!prepared.text_request.decode_options.skip_special_tokens);
assert_eq!(
prepared.text_request.sampling_params.stream_interval,
NonZeroU32::new(4)
);
}
#[test]
@@ -2,6 +2,7 @@
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::collections::HashMap;
use std::num::NonZeroU32;
use std::sync::Arc;
use serde::{Deserialize, Serialize};
@@ -114,6 +115,12 @@ pub struct CompletionRequest {
/// Parameters for detecting repetitive N-gram patterns in output tokens
pub repetition_detection: Option<RepetitionDetectionParams>,
/// Number of tokens to batch into each streamed chunk. Raises the server's
/// `--stream-interval` for this request. Values below the server setting
/// are clamped up to it. The first and last chunks are always sent
/// immediately. Ignored for non-streaming requests.
pub stream_interval: Option<NonZeroU32>,
/// Length penalty for beam search
pub length_penalty: Option<f32>,