[Rust Frontend] Support --stream-interval and per-request stream_interval (#60589)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
ffdde24f54
commit
35e920d671
@@ -124,6 +124,7 @@ pub fn lower_sampling_params(
|
||||
structured_outputs,
|
||||
skip_reading_prefix_cache,
|
||||
vllm_xargs,
|
||||
stream_interval,
|
||||
} = sampling_params;
|
||||
|
||||
validate_logprobs(
|
||||
@@ -214,6 +215,7 @@ pub fn lower_sampling_params(
|
||||
skip_reading_prefix_cache,
|
||||
extra_args: vllm_xargs,
|
||||
routed_experts_prompt_start: 0,
|
||||
stream_interval,
|
||||
};
|
||||
validate_resolved_sampling_params(¶ms)?;
|
||||
validate_vocab_range(¶ms, &sampling_limits)?;
|
||||
@@ -341,6 +343,7 @@ fn merge_unique_token_ids(
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::{BTreeSet, HashMap};
|
||||
use std::num::NonZeroU32;
|
||||
|
||||
use serial_test::file_serial;
|
||||
use vllm_engine_core_client::protocol::kv_hints::{KvHintAction, KvHintsEnvelope};
|
||||
@@ -693,6 +696,7 @@ mod tests {
|
||||
skip_reading_prefix_cache: None,
|
||||
extra_args: None,
|
||||
routed_experts_prompt_start: 0,
|
||||
stream_interval: None,
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(¶ms);
|
||||
@@ -746,6 +750,7 @@ mod tests {
|
||||
skip_reading_prefix_cache: None,
|
||||
extra_args: None,
|
||||
routed_experts_prompt_start: 0,
|
||||
stream_interval: None,
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(¶ms);
|
||||
@@ -921,6 +926,7 @@ mod tests {
|
||||
skip_reading_prefix_cache: None,
|
||||
extra_args: None,
|
||||
routed_experts_prompt_start: 0,
|
||||
stream_interval: None,
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(¶ms);
|
||||
@@ -992,6 +998,7 @@ mod tests {
|
||||
skip_reading_prefix_cache: None,
|
||||
extra_args: None,
|
||||
routed_experts_prompt_start: 0,
|
||||
stream_interval: None,
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(¶ms);
|
||||
@@ -1056,6 +1063,7 @@ mod tests {
|
||||
skip_reading_prefix_cache: None,
|
||||
extra_args: None,
|
||||
routed_experts_prompt_start: 0,
|
||||
stream_interval: None,
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(¶ms);
|
||||
@@ -1401,6 +1409,7 @@ mod tests {
|
||||
skip_reading_prefix_cache: None,
|
||||
extra_args: None,
|
||||
routed_experts_prompt_start: 0,
|
||||
stream_interval: None,
|
||||
}
|
||||
"#]]
|
||||
.assert_debug_eq(¶ms);
|
||||
@@ -1430,6 +1439,26 @@ mod tests {
|
||||
assert_eq!(prepared.generate_request.request_id, "text-1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_text_request_carries_stream_interval_in_sampling_params() {
|
||||
let mut request = sample_request();
|
||||
request.sampling_params.stream_interval = NonZeroU32::new(4);
|
||||
|
||||
let prepared = lower_text_request(
|
||||
request,
|
||||
vec![1, 2, 3],
|
||||
sample_sampling_hints(),
|
||||
sample_sampling_limits(),
|
||||
&stub_tokenizer(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
prepared.generate_request.sampling_params.stream_interval,
|
||||
NonZeroU32::new(4)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_text_request_passes_kv_hints_through() {
|
||||
let hints = KvHintsEnvelope {
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::num::NonZeroU32;
|
||||
|
||||
use enum_as_inner::EnumAsInner;
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -147,6 +148,11 @@ pub struct SamplingParams {
|
||||
pub skip_reading_prefix_cache: Option<bool>,
|
||||
/// Additional request parameters for custom extensions.
|
||||
pub vllm_xargs: Option<HashMap<String, Value>>,
|
||||
/// Number of newly generated tokens to batch into each streamed output.
|
||||
/// Raises the interval above the frontend-level `--stream-interval`;
|
||||
/// values below it are clamped up to it. The first and final outputs are
|
||||
/// always emitted immediately.
|
||||
pub stream_interval: Option<NonZeroU32>,
|
||||
}
|
||||
|
||||
#[allow(clippy::derivable_impls)] // more explicit
|
||||
@@ -180,6 +186,7 @@ impl Default for SamplingParams {
|
||||
structured_outputs: None,
|
||||
skip_reading_prefix_cache: None,
|
||||
vllm_xargs: None,
|
||||
stream_interval: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user