[Rust Frontend] Support --stream-interval and per-request stream_interval (#60589)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
ffdde24f54
commit
35e920d671
@@ -2,6 +2,7 @@
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::num::NonZeroU32;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
@@ -81,6 +82,7 @@ async fn main() -> Result<()> {
|
||||
lora_modules: Vec::new(),
|
||||
chat_template_content_format: ChatTemplateContentFormatOption::Auto,
|
||||
max_logprobs: None,
|
||||
stream_interval: NonZeroU32::MIN,
|
||||
api_server_options: ApiServerOptions::default(),
|
||||
cors: CorsConfig::default(),
|
||||
tls: None,
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fmt;
|
||||
use std::num::NonZeroU32;
|
||||
use std::str::FromStr;
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -258,6 +259,10 @@ pub struct Config {
|
||||
/// Optional maximum number of top log probabilities accepted by the
|
||||
/// frontend. `None` delegates to the text layer default.
|
||||
pub max_logprobs: Option<i32>,
|
||||
/// Minimum number of newly generated tokens batched into each streamed
|
||||
/// output after the first one. Requests can raise it with their own
|
||||
/// `stream_interval`.
|
||||
pub stream_interval: NonZeroU32,
|
||||
/// HTTP/API-server behavior switches.
|
||||
pub api_server_options: ApiServerOptions,
|
||||
/// CORS settings applied to every HTTP response.
|
||||
|
||||
@@ -123,7 +123,9 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
|
||||
.await
|
||||
.context("failed to connect to engine core")?;
|
||||
|
||||
let llm = Llm::new(client).with_log_stats(!config.disable_log_stats);
|
||||
let llm = Llm::new(client)
|
||||
.with_log_stats(!config.disable_log_stats)
|
||||
.with_stream_interval(config.stream_interval);
|
||||
let text = TextLlm::new(llm, text_backend).with_max_logprobs(config.max_logprobs);
|
||||
|
||||
let chat = ChatLlm::new(text, chat_backend)
|
||||
|
||||
@@ -179,6 +179,7 @@ pub(super) fn prepare_chat_request(
|
||||
merge_ec_transfer_params(request.vllm_xargs, request.ec_transfer_params.as_ref()),
|
||||
request.kv_transfer_params.as_ref(),
|
||||
),
|
||||
stream_interval: request.stream_interval,
|
||||
},
|
||||
chat_options: ChatOptions {
|
||||
generation_prompt_mode,
|
||||
@@ -443,6 +444,7 @@ fn convert_tool_choice(tool_choice: &ToolChoice) -> Result<ChatToolChoice, ApiEr
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::HashMap;
|
||||
use std::num::NonZeroU32;
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::http::{HeaderMap, StatusCode};
|
||||
@@ -540,6 +542,15 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_http_request_rejects_zero_stream_interval() {
|
||||
let request = json!({
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"stream_interval": 0,
|
||||
});
|
||||
assert!(serde_json::from_value::<ChatCompletionRequest>(request).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_http_request_defaults_missing_or_null_model() {
|
||||
for model in [None, Some(serde_json::Value::Null)] {
|
||||
@@ -1004,6 +1015,7 @@ mod tests {
|
||||
frequency_penalty: Some(0.3),
|
||||
presence_penalty: Some(0.4),
|
||||
repetition_penalty: Some(1.1),
|
||||
stream_interval: NonZeroU32::new(4),
|
||||
..base_request()
|
||||
};
|
||||
|
||||
@@ -1019,6 +1031,7 @@ mod tests {
|
||||
frequency_penalty: Some(0.3),
|
||||
presence_penalty: Some(0.4),
|
||||
repetition_penalty: Some(1.1),
|
||||
stream_interval: NonZeroU32::new(4),
|
||||
..VllmSamplingParams::default()
|
||||
};
|
||||
assert_eq!(prepared.chat_request.sampling_params, expected);
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fmt;
|
||||
use std::num::NonZeroU32;
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -256,6 +257,12 @@ pub struct ChatCompletionRequest {
|
||||
|
||||
/// Parameters for detecting repetitive N-gram patterns in output tokens
|
||||
pub repetition_detection: Option<RepetitionDetectionParams>,
|
||||
|
||||
/// Number of tokens to batch into each streamed chunk. Raises the server's
|
||||
/// `--stream-interval` for this request. Values below the server setting
|
||||
/// are clamped up to it. The first and last chunks are always sent
|
||||
/// immediately. Ignored for non-streaming requests.
|
||||
pub stream_interval: Option<NonZeroU32>,
|
||||
}
|
||||
|
||||
impl Default for ChatCompletionRequest {
|
||||
@@ -323,6 +330,7 @@ impl Default for ChatCompletionRequest {
|
||||
ec_transfer_params: None,
|
||||
vllm_xargs: None,
|
||||
repetition_detection: None,
|
||||
stream_interval: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -144,6 +144,7 @@ pub(super) fn prepare_completion_request(
|
||||
merge_ec_transfer_params(request.vllm_xargs, request.ec_transfer_params.as_ref()),
|
||||
request.kv_transfer_params.as_ref(),
|
||||
),
|
||||
stream_interval: request.stream_interval,
|
||||
},
|
||||
decode_options: TextDecodeOptions {
|
||||
skip_special_tokens: request.skip_special_tokens,
|
||||
@@ -211,6 +212,8 @@ fn completion_echo_text(
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::num::NonZeroU32;
|
||||
|
||||
use axum::http::HeaderMap;
|
||||
use serde_json::json;
|
||||
use validator::Validate;
|
||||
@@ -400,7 +403,8 @@ mod tests {
|
||||
"presence_penalty": 0.3,
|
||||
"repetition_penalty": 1.1,
|
||||
"ignore_eos": true,
|
||||
"skip_special_tokens": false
|
||||
"skip_special_tokens": false,
|
||||
"stream_interval": 4
|
||||
}))
|
||||
.expect("parse request");
|
||||
|
||||
@@ -436,6 +440,10 @@ mod tests {
|
||||
);
|
||||
assert!(prepared.text_request.sampling_params.ignore_eos);
|
||||
assert!(!prepared.text_request.decode_options.skip_special_tokens);
|
||||
assert_eq!(
|
||||
prepared.text_request.sampling_params.stream_interval,
|
||||
NonZeroU32::new(4)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::num::NonZeroU32;
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -114,6 +115,12 @@ pub struct CompletionRequest {
|
||||
/// Parameters for detecting repetitive N-gram patterns in output tokens
|
||||
pub repetition_detection: Option<RepetitionDetectionParams>,
|
||||
|
||||
/// Number of tokens to batch into each streamed chunk. Raises the server's
|
||||
/// `--stream-interval` for this request. Values below the server setting
|
||||
/// are clamped up to it. The first and last chunks are always sent
|
||||
/// immediately. Ignored for non-streaming requests.
|
||||
pub stream_interval: Option<NonZeroU32>,
|
||||
|
||||
/// Length penalty for beam search
|
||||
pub length_penalty: Option<f32>,
|
||||
|
||||
|
||||
Reference in New Issue
Block a user