[Rust Frontend] add repetition_detection support to sampling params (#46684)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Bugen Zhao
parent
fbc9ba6d30
commit
6429d5f527
@@ -27,6 +27,8 @@ pub enum Error {
|
||||
MinTokensExceedsMaxTokens { min_tokens: u32, max_tokens: u32 },
|
||||
#[error("`thinking_token_budget` must be a non-negative integer or -1 for unlimited.")]
|
||||
InvalidThinkingTokenBudget,
|
||||
#[error("invalid repetition detection params: {message}")]
|
||||
InvalidRepetitionDetection { message: String },
|
||||
#[error("text request stream `{request_id}` closed before terminal output")]
|
||||
StreamClosedBeforeTerminalOutput { request_id: String },
|
||||
#[error(transparent)]
|
||||
@@ -47,6 +49,7 @@ impl Error {
|
||||
| Self::TokenIds(_)
|
||||
| Self::MinTokensExceedsMaxTokens { .. }
|
||||
| Self::InvalidThinkingTokenBudget
|
||||
| Self::InvalidRepetitionDetection { .. }
|
||||
// An empty tokenized prompt detected later, at request prepare
|
||||
// time, surfaces through the transparent Llm wrapper.
|
||||
| Self::Llm(LlmError::EmptyPromptTokenIds { .. }) => true,
|
||||
|
||||
@@ -5,7 +5,9 @@ pub(crate) mod token_ids;
|
||||
|
||||
use logprobs::validate_logprobs;
|
||||
use token_ids::{validate_prompt_token_ids, validate_vocab_range};
|
||||
use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams;
|
||||
use vllm_engine_core_client::protocol::sampling::{
|
||||
EngineCoreSamplingParams, RepetitionDetectionParams,
|
||||
};
|
||||
use vllm_llm::GenerateRequest;
|
||||
use vllm_tokenizer::Tokenizer;
|
||||
|
||||
@@ -93,6 +95,7 @@ pub fn lower_sampling_params(
|
||||
frequency_penalty,
|
||||
presence_penalty,
|
||||
repetition_penalty,
|
||||
repetition_detection,
|
||||
stop_token_ids,
|
||||
ignore_eos,
|
||||
logit_bias,
|
||||
@@ -110,6 +113,7 @@ pub fn lower_sampling_params(
|
||||
logprob_token_ids.as_deref(),
|
||||
sampling_limits,
|
||||
)?;
|
||||
validate_repetition_detection(repetition_detection.as_ref())?;
|
||||
|
||||
// Mirrors the model-generation-config inheritance used by vLLM's OpenAI chat
|
||||
// path: https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/entrypoints/openai/chat_completion/protocol.py#L424-L450
|
||||
@@ -163,6 +167,7 @@ pub fn lower_sampling_params(
|
||||
frequency_penalty,
|
||||
presence_penalty,
|
||||
repetition_penalty,
|
||||
repetition_detection: repetition_detection.filter(|p| !p.is_disabled()),
|
||||
stop_token_ids,
|
||||
eos_token_id: (!ignore_eos).then_some(primary_eos_token_id).flatten(),
|
||||
all_stop_token_ids,
|
||||
@@ -194,6 +199,33 @@ fn normalize_thinking_token_budget(value: Option<i64>) -> Result<Option<u64>> {
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_repetition_detection(params: Option<&RepetitionDetectionParams>) -> Result<()> {
|
||||
let Some(params) = params else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
if params.min_pattern_size > params.max_pattern_size {
|
||||
return Err(Error::InvalidRepetitionDetection {
|
||||
message: format!(
|
||||
"`min_pattern_size` must be less than or equal to \
|
||||
`max_pattern_size`, got min_pattern_size={}, \
|
||||
max_pattern_size={}",
|
||||
params.min_pattern_size, params.max_pattern_size
|
||||
),
|
||||
});
|
||||
}
|
||||
if params.max_pattern_size > 0 && params.min_count < 2 {
|
||||
return Err(Error::InvalidRepetitionDetection {
|
||||
message: format!(
|
||||
"`min_count` must be at least 2, got min_count={}",
|
||||
params.min_count
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Convert bad-word strings into token-ID sequences, following the Python vLLM
|
||||
/// logic in `SamplingParams.update_from_tokenizer()`.
|
||||
///
|
||||
@@ -390,6 +422,59 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_sampling_params_validates_repetition_detection() {
|
||||
let lower = |repetition_detection| {
|
||||
lower_sampling_params_with_limits(
|
||||
SamplingParams {
|
||||
repetition_detection,
|
||||
..SamplingParams::default()
|
||||
},
|
||||
sample_sampling_limits(),
|
||||
)
|
||||
};
|
||||
|
||||
let enabled = RepetitionDetectionParams {
|
||||
max_pattern_size: 4,
|
||||
min_pattern_size: 2,
|
||||
min_count: 2,
|
||||
};
|
||||
assert_eq!(
|
||||
lower(Some(enabled.clone())).unwrap().repetition_detection,
|
||||
Some(enabled)
|
||||
);
|
||||
|
||||
let disabled = RepetitionDetectionParams {
|
||||
max_pattern_size: 0,
|
||||
min_pattern_size: 0,
|
||||
min_count: 0,
|
||||
};
|
||||
assert_eq!(lower(Some(disabled)).unwrap().repetition_detection, None);
|
||||
|
||||
let error = lower(Some(RepetitionDetectionParams {
|
||||
max_pattern_size: 1,
|
||||
min_pattern_size: 2,
|
||||
min_count: 2,
|
||||
}))
|
||||
.unwrap_err();
|
||||
let Error::InvalidRepetitionDetection { message } = error else {
|
||||
panic!("expected repetition_detection validation error");
|
||||
};
|
||||
assert!(message.contains("min_pattern_size=2"));
|
||||
assert!(message.contains("max_pattern_size=1"));
|
||||
|
||||
let error = lower(Some(RepetitionDetectionParams {
|
||||
max_pattern_size: 1,
|
||||
min_pattern_size: 1,
|
||||
min_count: 1,
|
||||
}))
|
||||
.unwrap_err();
|
||||
let Error::InvalidRepetitionDetection { message } = error else {
|
||||
panic!("expected repetition_detection validation error");
|
||||
};
|
||||
assert!(message.contains("min_count=1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_text_request_applies_python_style_eos_hints() {
|
||||
let prepared = lower_text_request(
|
||||
@@ -417,6 +502,7 @@ mod tests {
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
repetition_penalty: 1.0,
|
||||
repetition_detection: None,
|
||||
stop_token_ids: [
|
||||
77,
|
||||
],
|
||||
@@ -469,6 +555,7 @@ mod tests {
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
repetition_penalty: 1.0,
|
||||
repetition_detection: None,
|
||||
stop_token_ids: [],
|
||||
eos_token_id: None,
|
||||
all_stop_token_ids: {
|
||||
@@ -600,6 +687,7 @@ mod tests {
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
repetition_penalty: 1.0,
|
||||
repetition_detection: None,
|
||||
stop_token_ids: [
|
||||
151643,
|
||||
],
|
||||
@@ -662,6 +750,7 @@ mod tests {
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
repetition_penalty: 1.0,
|
||||
repetition_detection: None,
|
||||
stop_token_ids: [
|
||||
11,
|
||||
77,
|
||||
@@ -732,6 +821,7 @@ mod tests {
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
repetition_penalty: 1.2,
|
||||
repetition_detection: None,
|
||||
stop_token_ids: [],
|
||||
eos_token_id: None,
|
||||
all_stop_token_ids: {},
|
||||
@@ -980,6 +1070,7 @@ mod tests {
|
||||
frequency_penalty: 0.0,
|
||||
presence_penalty: 0.0,
|
||||
repetition_penalty: 1.2,
|
||||
repetition_detection: None,
|
||||
stop_token_ids: [],
|
||||
eos_token_id: None,
|
||||
all_stop_token_ids: {},
|
||||
|
||||
@@ -6,6 +6,7 @@ use serde_json::Value;
|
||||
use vllm_engine_core_client::protocol::lora::LoraRequest;
|
||||
use vllm_engine_core_client::protocol::multimodal::MmFeatures;
|
||||
use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
|
||||
use vllm_engine_core_client::protocol::sampling::RepetitionDetectionParams;
|
||||
use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsParams;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
@@ -83,6 +84,9 @@ pub struct SamplingParams {
|
||||
/// Repetition penalty applied by the sampler. `None` means no explicit user
|
||||
/// override.
|
||||
pub repetition_penalty: Option<f32>,
|
||||
/// Parameters for detecting repetitive N-gram patterns. `None` means no
|
||||
/// explicit user override.
|
||||
pub repetition_detection: Option<RepetitionDetectionParams>,
|
||||
/// Explicit stop token IDs provided by the caller. `None` means no explicit
|
||||
/// user override.
|
||||
pub stop_token_ids: Option<Vec<u32>>,
|
||||
@@ -130,6 +134,7 @@ impl Default for SamplingParams {
|
||||
frequency_penalty: None,
|
||||
presence_penalty: None,
|
||||
repetition_penalty: None,
|
||||
repetition_detection: None,
|
||||
stop_token_ids: None,
|
||||
ignore_eos: false,
|
||||
logit_bias: None,
|
||||
|
||||
Reference in New Issue
Block a user