[Rust Frontend] add repetition_detection support to sampling params (#46684)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
yangyang-cs95
2026-07-03 14:06:23 +08:00
committed by GitHub
co-authored by Bugen Zhao
parent fbc9ba6d30
commit 6429d5f527
14 changed files with 177 additions and 20 deletions
+3
View File
@@ -27,6 +27,8 @@ pub enum Error {
MinTokensExceedsMaxTokens { min_tokens: u32, max_tokens: u32 },
#[error("`thinking_token_budget` must be a non-negative integer or -1 for unlimited.")]
InvalidThinkingTokenBudget,
#[error("invalid repetition detection params: {message}")]
InvalidRepetitionDetection { message: String },
#[error("text request stream `{request_id}` closed before terminal output")]
StreamClosedBeforeTerminalOutput { request_id: String },
#[error(transparent)]
@@ -47,6 +49,7 @@ impl Error {
| Self::TokenIds(_)
| Self::MinTokensExceedsMaxTokens { .. }
| Self::InvalidThinkingTokenBudget
| Self::InvalidRepetitionDetection { .. }
// An empty tokenized prompt detected later, at request prepare
// time, surfaces through the transparent Llm wrapper.
| Self::Llm(LlmError::EmptyPromptTokenIds { .. }) => true,
+92 -1
View File
@@ -5,7 +5,9 @@ pub(crate) mod token_ids;
use logprobs::validate_logprobs;
use token_ids::{validate_prompt_token_ids, validate_vocab_range};
use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams;
use vllm_engine_core_client::protocol::sampling::{
EngineCoreSamplingParams, RepetitionDetectionParams,
};
use vllm_llm::GenerateRequest;
use vllm_tokenizer::Tokenizer;
@@ -93,6 +95,7 @@ pub fn lower_sampling_params(
frequency_penalty,
presence_penalty,
repetition_penalty,
repetition_detection,
stop_token_ids,
ignore_eos,
logit_bias,
@@ -110,6 +113,7 @@ pub fn lower_sampling_params(
logprob_token_ids.as_deref(),
sampling_limits,
)?;
validate_repetition_detection(repetition_detection.as_ref())?;
// Mirrors the model-generation-config inheritance used by vLLM's OpenAI chat
// path: https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/entrypoints/openai/chat_completion/protocol.py#L424-L450
@@ -163,6 +167,7 @@ pub fn lower_sampling_params(
frequency_penalty,
presence_penalty,
repetition_penalty,
repetition_detection: repetition_detection.filter(|p| !p.is_disabled()),
stop_token_ids,
eos_token_id: (!ignore_eos).then_some(primary_eos_token_id).flatten(),
all_stop_token_ids,
@@ -194,6 +199,33 @@ fn normalize_thinking_token_budget(value: Option<i64>) -> Result<Option<u64>> {
}
}
fn validate_repetition_detection(params: Option<&RepetitionDetectionParams>) -> Result<()> {
let Some(params) = params else {
return Ok(());
};
if params.min_pattern_size > params.max_pattern_size {
return Err(Error::InvalidRepetitionDetection {
message: format!(
"`min_pattern_size` must be less than or equal to \
`max_pattern_size`, got min_pattern_size={}, \
max_pattern_size={}",
params.min_pattern_size, params.max_pattern_size
),
});
}
if params.max_pattern_size > 0 && params.min_count < 2 {
return Err(Error::InvalidRepetitionDetection {
message: format!(
"`min_count` must be at least 2, got min_count={}",
params.min_count
),
});
}
Ok(())
}
/// Convert bad-word strings into token-ID sequences, following the Python vLLM
/// logic in `SamplingParams.update_from_tokenizer()`.
///
@@ -390,6 +422,59 @@ mod tests {
));
}
#[test]
fn lower_sampling_params_validates_repetition_detection() {
let lower = |repetition_detection| {
lower_sampling_params_with_limits(
SamplingParams {
repetition_detection,
..SamplingParams::default()
},
sample_sampling_limits(),
)
};
let enabled = RepetitionDetectionParams {
max_pattern_size: 4,
min_pattern_size: 2,
min_count: 2,
};
assert_eq!(
lower(Some(enabled.clone())).unwrap().repetition_detection,
Some(enabled)
);
let disabled = RepetitionDetectionParams {
max_pattern_size: 0,
min_pattern_size: 0,
min_count: 0,
};
assert_eq!(lower(Some(disabled)).unwrap().repetition_detection, None);
let error = lower(Some(RepetitionDetectionParams {
max_pattern_size: 1,
min_pattern_size: 2,
min_count: 2,
}))
.unwrap_err();
let Error::InvalidRepetitionDetection { message } = error else {
panic!("expected repetition_detection validation error");
};
assert!(message.contains("min_pattern_size=2"));
assert!(message.contains("max_pattern_size=1"));
let error = lower(Some(RepetitionDetectionParams {
max_pattern_size: 1,
min_pattern_size: 1,
min_count: 1,
}))
.unwrap_err();
let Error::InvalidRepetitionDetection { message } = error else {
panic!("expected repetition_detection validation error");
};
assert!(message.contains("min_count=1"));
}
#[test]
fn lower_text_request_applies_python_style_eos_hints() {
let prepared = lower_text_request(
@@ -417,6 +502,7 @@ mod tests {
frequency_penalty: 0.0,
presence_penalty: 0.0,
repetition_penalty: 1.0,
repetition_detection: None,
stop_token_ids: [
77,
],
@@ -469,6 +555,7 @@ mod tests {
frequency_penalty: 0.0,
presence_penalty: 0.0,
repetition_penalty: 1.0,
repetition_detection: None,
stop_token_ids: [],
eos_token_id: None,
all_stop_token_ids: {
@@ -600,6 +687,7 @@ mod tests {
frequency_penalty: 0.0,
presence_penalty: 0.0,
repetition_penalty: 1.0,
repetition_detection: None,
stop_token_ids: [
151643,
],
@@ -662,6 +750,7 @@ mod tests {
frequency_penalty: 0.0,
presence_penalty: 0.0,
repetition_penalty: 1.0,
repetition_detection: None,
stop_token_ids: [
11,
77,
@@ -732,6 +821,7 @@ mod tests {
frequency_penalty: 0.0,
presence_penalty: 0.0,
repetition_penalty: 1.2,
repetition_detection: None,
stop_token_ids: [],
eos_token_id: None,
all_stop_token_ids: {},
@@ -980,6 +1070,7 @@ mod tests {
frequency_penalty: 0.0,
presence_penalty: 0.0,
repetition_penalty: 1.2,
repetition_detection: None,
stop_token_ids: [],
eos_token_id: None,
all_stop_token_ids: {},
+5
View File
@@ -6,6 +6,7 @@ use serde_json::Value;
use vllm_engine_core_client::protocol::lora::LoraRequest;
use vllm_engine_core_client::protocol::multimodal::MmFeatures;
use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
use vllm_engine_core_client::protocol::sampling::RepetitionDetectionParams;
use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsParams;
use crate::error::{Error, Result};
@@ -83,6 +84,9 @@ pub struct SamplingParams {
/// Repetition penalty applied by the sampler. `None` means no explicit user
/// override.
pub repetition_penalty: Option<f32>,
/// Parameters for detecting repetitive N-gram patterns. `None` means no
/// explicit user override.
pub repetition_detection: Option<RepetitionDetectionParams>,
/// Explicit stop token IDs provided by the caller. `None` means no explicit
/// user override.
pub stop_token_ids: Option<Vec<u32>>,
@@ -130,6 +134,7 @@ impl Default for SamplingParams {
frequency_penalty: None,
presence_penalty: None,
repetition_penalty: None,
repetition_detection: None,
stop_token_ids: None,
ignore_eos: false,
logit_bias: None,