[Rust Frontend] add repetition_detection support to sampling params (#46684)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
yangyang-cs95
2026-07-03 14:06:23 +08:00
committed by GitHub
co-authored by Bugen Zhao
parent fbc9ba6d30
commit 6429d5f527
14 changed files with 177 additions and 20 deletions
@@ -21,6 +21,30 @@ fn default_max_tokens() -> u32 {
16 16
} }
///
/// Parameters for detecting repetitive N-gram patterns in output tokens.
///
/// Mirrors Python's `RepetitionDetectionParams`:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/sampling_params.py#L109-L144>
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct RepetitionDetectionParams {
/// Maximum N-gram size to check. 0 disables detection.
pub max_pattern_size: u32,
/// Minimum N-gram size to check. Defaults to 1 when zero.
#[serde(default)]
pub min_pattern_size: u32,
/// Minimum number of repetitions to trigger detection (must be >= 2).
pub min_count: u32,
}
impl RepetitionDetectionParams {
/// Return `true` when the params are effectively disabled (max_pattern_size
/// is 0).
pub fn is_disabled(&self) -> bool {
self.max_pattern_size == 0
}
}
/// Engine-core-facing sampling parameters for text generation. /// Engine-core-facing sampling parameters for text generation.
/// ///
/// This is the normalized southbound subset used by the Rust frontend when it /// This is the normalized southbound subset used by the Rust frontend when it
@@ -76,6 +100,9 @@ pub struct EngineCoreSamplingParams {
/// Repetition penalty applied by the sampler. /// Repetition penalty applied by the sampler.
#[serde(default = "default_repetition_penalty")] #[serde(default = "default_repetition_penalty")]
pub repetition_penalty: f32, pub repetition_penalty: f32,
/// Parameters for detecting repetitive N-gram patterns. `None` disables
/// detection.
pub repetition_detection: Option<RepetitionDetectionParams>,
/// Token IDs that stop generation. /// Token IDs that stop generation.
pub stop_token_ids: Vec<u32>, pub stop_token_ids: Vec<u32>,
/// Primary EOS token ID used by engine-core's dedicated EOS stop path. /// Primary EOS token ID used by engine-core's dedicated EOS stop path.
@@ -134,6 +161,7 @@ impl EngineCoreSamplingParams {
frequency_penalty: 0.0, frequency_penalty: 0.0,
presence_penalty: 0.0, presence_penalty: 0.0,
repetition_penalty: 1.0, repetition_penalty: 1.0,
repetition_detection: None,
stop_token_ids: Vec::new(), stop_token_ids: Vec::new(),
eos_token_id: None, eos_token_id: None,
all_stop_token_ids: BTreeSet::new(), all_stop_token_ids: BTreeSet::new(),
@@ -203,6 +231,7 @@ mod tests {
assert_eq!(sampling.frequency_penalty, 0.0); assert_eq!(sampling.frequency_penalty, 0.0);
assert_eq!(sampling.presence_penalty, 0.0); assert_eq!(sampling.presence_penalty, 0.0);
assert_eq!(sampling.repetition_penalty, 1.0); assert_eq!(sampling.repetition_penalty, 1.0);
assert_eq!(sampling.repetition_detection, None);
assert_eq!(sampling.logprobs, None); assert_eq!(sampling.logprobs, None);
assert_eq!(sampling.prompt_logprobs, None); assert_eq!(sampling.prompt_logprobs, None);
assert_eq!(sampling.eos_token_id, None); assert_eq!(sampling.eos_token_id, None);
@@ -2462,6 +2462,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
frequency_penalty: 0.0, frequency_penalty: 0.0,
presence_penalty: 0.0, presence_penalty: 0.0,
repetition_penalty: 1.0, repetition_penalty: 1.0,
repetition_detection: None,
stop_token_ids: Vec::new(), stop_token_ids: Vec::new(),
eos_token_id: None, eos_token_id: None,
all_stop_token_ids: BTreeSet::new(), all_stop_token_ids: BTreeSet::new(),
+5 -3
View File
@@ -69,7 +69,7 @@ pub enum FinishReason {
/// A retryable request-level internal error occurred. /// A retryable request-level internal error occurred.
Error, Error,
/// A repetitive token pattern was detected. /// A repetitive token pattern was detected.
Repetition, Repetition(Option<StopReason>),
} }
impl FinishReason { impl FinishReason {
@@ -87,7 +87,7 @@ impl FinishReason {
Self::Length => "length", Self::Length => "length",
Self::Abort => "abort", Self::Abort => "abort",
Self::Error => "error", Self::Error => "error",
Self::Repetition => "repetition", Self::Repetition(_) => "repetition",
} }
} }
@@ -96,6 +96,7 @@ impl FinishReason {
pub fn as_stop_reason(&self) -> Option<&StopReason> { pub fn as_stop_reason(&self) -> Option<&StopReason> {
match self { match self {
Self::Stop(stop_reason) => stop_reason.as_ref(), Self::Stop(stop_reason) => stop_reason.as_ref(),
Self::Repetition(stop_reason) => stop_reason.as_ref(),
_ => None, _ => None,
} }
} }
@@ -105,6 +106,7 @@ impl FinishReason {
pub fn into_stop_reason(self) -> Option<StopReason> { pub fn into_stop_reason(self) -> Option<StopReason> {
match self { match self {
Self::Stop(stop_reason) => stop_reason, Self::Stop(stop_reason) => stop_reason,
Self::Repetition(stop_reason) => stop_reason,
_ => None, _ => None,
} }
} }
@@ -119,7 +121,7 @@ fn finish_reason_from_engine(
EngineCoreFinishReason::Length => FinishReason::Length, EngineCoreFinishReason::Length => FinishReason::Length,
EngineCoreFinishReason::Abort => FinishReason::Abort, EngineCoreFinishReason::Abort => FinishReason::Abort,
EngineCoreFinishReason::Error => FinishReason::Error, EngineCoreFinishReason::Error => FinishReason::Error,
EngineCoreFinishReason::Repetition => FinishReason::Repetition, EngineCoreFinishReason::Repetition => FinishReason::Repetition(stop_reason),
}) })
} }
+1 -1
View File
@@ -332,7 +332,7 @@ fn to_finish_info(finished: &Finished, token_ids: &[u32]) -> pb::FinishInfo {
(PbFinishReason::Stop as i32, sr) (PbFinishReason::Stop as i32, sr)
} }
FinishReason::Length => (PbFinishReason::Length as i32, None), FinishReason::Length => (PbFinishReason::Length as i32, None),
FinishReason::Abort | FinishReason::Error | FinishReason::Repetition => { FinishReason::Abort | FinishReason::Error | FinishReason::Repetition(_) => {
(PbFinishReason::Aborted as i32, None) (PbFinishReason::Aborted as i32, None)
} }
}; };
@@ -805,7 +805,7 @@ fn chat_finish_reason_to_openai(
FinishReason::Stop(_) => Ok("stop"), FinishReason::Stop(_) => Ok("stop"),
FinishReason::Length => Ok("length"), FinishReason::Length => Ok("length"),
FinishReason::Abort => Ok("abort"), FinishReason::Abort => Ok("abort"),
FinishReason::Repetition => Ok("stop"), FinishReason::Repetition(_) => Ok("repetition"),
FinishReason::Error => { FinishReason::Error => {
bail_server_error!("Internal server error"); bail_server_error!("Internal server error");
} }
@@ -122,6 +122,7 @@ pub(super) fn prepare_chat_request(
frequency_penalty: request.frequency_penalty, frequency_penalty: request.frequency_penalty,
presence_penalty: request.presence_penalty, presence_penalty: request.presence_penalty,
repetition_penalty: request.repetition_penalty, repetition_penalty: request.repetition_penalty,
repetition_detection: request.repetition_detection,
stop_token_ids: request.stop_token_ids, stop_token_ids: request.stop_token_ids,
ignore_eos: request.ignore_eos, ignore_eos: request.ignore_eos,
logit_bias: convert_logit_bias(request.logit_bias)?, logit_bias: convert_logit_bias(request.logit_bias)?,
@@ -6,6 +6,7 @@ use serde_json::Value;
use serde_with::SerializeDisplay; use serde_with::SerializeDisplay;
use validator::Validate; use validator::Validate;
use vllm_chat::ReasoningEffort; use vllm_chat::ReasoningEffort;
use vllm_engine_core_client::protocol::sampling::RepetitionDetectionParams;
use crate::routes::openai::utils::structured_outputs::ResponseFormat; use crate::routes::openai::utils::structured_outputs::ResponseFormat;
use crate::routes::openai::utils::types::{ use crate::routes::openai::utils::types::{
@@ -236,7 +237,7 @@ pub struct ChatCompletionRequest {
pub vllm_xargs: Option<HashMap<String, Value>>, pub vllm_xargs: Option<HashMap<String, Value>>,
/// Parameters for detecting repetitive N-gram patterns in output tokens /// Parameters for detecting repetitive N-gram patterns in output tokens
pub repetition_detection: Option<Value>, pub repetition_detection: Option<RepetitionDetectionParams>,
} }
impl Default for ChatCompletionRequest { impl Default for ChatCompletionRequest {
@@ -94,11 +94,6 @@ pub(super) fn validate_request_compat(
"mm_processor_kwargs", "mm_processor_kwargs",
"mm_processor_kwargs is not supported.", "mm_processor_kwargs is not supported.",
)?; )?;
reject_non_default(
request.repetition_detection.as_ref(),
"repetition_detection",
"repetition_detection is not supported.",
)?;
Ok(()) Ok(())
} }
@@ -14,9 +14,11 @@ use axum::http::HeaderMap;
use axum::response::sse::{Event, Sse}; use axum::response::sse::{Event, Sse};
use axum::response::{IntoResponse, Response}; use axum::response::{IntoResponse, Response};
use futures::{Stream, StreamExt as _, pin_mut}; use futures::{Stream, StreamExt as _, pin_mut};
use serde_json::Value;
use thiserror_ext::AsReport as _; use thiserror_ext::AsReport as _;
use tracing::{debug, error, info, trace}; use tracing::{debug, error, info, trace};
use tracing_futures::Instrument as _; use tracing_futures::Instrument as _;
use vllm_engine_core_client::protocol::output::StopReason;
use vllm_text::{ use vllm_text::{
DecodedPromptLogprobs, DecodedTextEvent, FinishReason, TextOutputStream, DecodedPromptLogprobs, DecodedTextEvent, FinishReason, TextOutputStream,
TextOutputStreamExt as _, TextOutputStreamExt as _,
@@ -142,9 +144,7 @@ async fn collect_completion(
.await .await
.map_err(|error| server_error!("completion stream failed: {}", error.to_report_string()))?; .map_err(|error| server_error!("completion stream failed: {}", error.to_report_string()))?;
let finish_reason = collected.finish_reason.clone(); let finish_reason = collected.finish_reason.clone();
let stop_reason = finish_reason let stop_reason = finish_reason.as_stop_reason().map(stop_reason_to_json);
.as_stop_reason()
.map(|sr| serde_json::to_value(sr).expect("StopReason must serialize to JSON"));
let prompt_char_count = echo.as_ref().map(|prompt| text_len(prompt)).unwrap_or_default(); let prompt_char_count = echo.as_ref().map(|prompt| text_len(prompt)).unwrap_or_default();
let logprobs = if requested_logprobs.is_some() && prompt_only { let logprobs = if requested_logprobs.is_some() && prompt_only {
@@ -181,7 +181,7 @@ async fn collect_completion(
Some(prompt) if prompt_only => prompt.clone(), Some(prompt) if prompt_only => prompt.clone(),
Some(prompt) => format!("{prompt}{}", collected.text), Some(prompt) => format!("{prompt}{}", collected.text),
}; };
let finish_reason = completion_finish_reason_to_openai(finish_reason)?.to_string(); let finish_reason = completion_finish_reason_to_openai(&finish_reason)?.to_string();
let usage = Usage::from_token_usage(collected.usage, enable_prompt_tokens_details); let usage = Usage::from_token_usage(collected.usage, enable_prompt_tokens_details);
if enable_log_requests { if enable_log_requests {
@@ -441,21 +441,24 @@ fn final_chunk(
created: u64, created: u64,
finish_reason: FinishReason, finish_reason: FinishReason,
) -> Result<CompletionStreamResponse, ApiError> { ) -> Result<CompletionStreamResponse, ApiError> {
let finish_reason = completion_finish_reason_to_openai(finish_reason)?; let stop_reason = finish_reason.as_stop_reason().map(stop_reason_to_json);
let finish_reason = completion_finish_reason_to_openai(&finish_reason)?;
let mut chunk = CompletionStreamResponse::new(request_id, response_model, created); let mut chunk = CompletionStreamResponse::new(request_id, response_model, created);
chunk.choices.push(CompletionStreamChoice { chunk.choices.push(CompletionStreamChoice {
finish_reason: Some(finish_reason.to_string()), finish_reason: Some(finish_reason.to_string()),
stop_reason,
..Default::default() ..Default::default()
}); });
Ok(chunk) Ok(chunk)
} }
fn completion_finish_reason_to_openai( fn completion_finish_reason_to_openai(
finish_reason: FinishReason, finish_reason: &FinishReason,
) -> Result<&'static str, ApiError> { ) -> Result<&'static str, ApiError> {
match finish_reason { match finish_reason {
FinishReason::Stop(_) | FinishReason::Repetition => Ok("stop"), FinishReason::Stop(_) => Ok("stop"),
FinishReason::Repetition(_) => Ok("repetition"),
FinishReason::Length => Ok("length"), FinishReason::Length => Ok("length"),
FinishReason::Abort => Ok("abort"), FinishReason::Abort => Ok("abort"),
FinishReason::Error => { FinishReason::Error => {
@@ -464,6 +467,10 @@ fn completion_finish_reason_to_openai(
} }
} }
fn stop_reason_to_json(stop_reason: &StopReason) -> Value {
serde_json::to_value(stop_reason).expect("StopReason must serialize to JSON")
}
fn prompt_only_logprobs_to_openai( fn prompt_only_logprobs_to_openai(
prompt_logprobs: Option<&DecodedPromptLogprobs>, prompt_logprobs: Option<&DecodedPromptLogprobs>,
prompt: &str, prompt: &str,
@@ -577,6 +584,7 @@ fn done_sse_event() -> Event {
mod tests { mod tests {
use futures::{StreamExt as _, stream}; use futures::{StreamExt as _, stream};
use itertools::Itertools as _; use itertools::Itertools as _;
use vllm_engine_core_client::protocol::output::StopReason;
use vllm_text::{ use vllm_text::{
DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedTextEvent, DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedTextEvent,
DecodedTokenLogprob, FinishReason, Finished, DecodedTokenLogprob, FinishReason, Finished,
@@ -670,7 +678,9 @@ mod tests {
output_token_count: 2, output_token_count: 2,
cached_token_count: 3, cached_token_count: 3,
}, },
finish_reason: FinishReason::stop_eos(), finish_reason: FinishReason::Repetition(Some(StopReason::Text(
"repetition_detected".to_string(),
))),
kv_transfer_params: None, kv_transfer_params: None,
}), }),
}), }),
@@ -726,6 +736,20 @@ mod tests {
CompletionSseChunk::Usage(_) => panic!("expected regular chunk"), CompletionSseChunk::Usage(_) => panic!("expected regular chunk"),
} }
match &chunks[2] {
CompletionSseChunk::Chunk(chunk) => {
assert_eq!(
chunk.choices[0].finish_reason.as_deref(),
Some("repetition")
);
assert_eq!(
chunk.choices[0].stop_reason,
Some(serde_json::json!("repetition_detected"))
);
}
CompletionSseChunk::Usage(_) => panic!("expected regular chunk"),
}
match &chunks[3] { match &chunks[3] {
CompletionSseChunk::Usage(chunk) => { CompletionSseChunk::Usage(chunk) => {
assert_eq!( assert_eq!(
@@ -117,6 +117,7 @@ pub(super) fn prepare_completion_request(
frequency_penalty: request.frequency_penalty, frequency_penalty: request.frequency_penalty,
presence_penalty: request.presence_penalty, presence_penalty: request.presence_penalty,
repetition_penalty: request.repetition_penalty, repetition_penalty: request.repetition_penalty,
repetition_detection: request.repetition_detection,
stop_token_ids: request.stop_token_ids, stop_token_ids: request.stop_token_ids,
ignore_eos: request.ignore_eos, ignore_eos: request.ignore_eos,
logit_bias: convert_logit_bias(request.logit_bias)?, logit_bias: convert_logit_bias(request.logit_bias)?,
@@ -3,6 +3,7 @@ use std::collections::HashMap;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use serde_json::{Map, Value}; use serde_json::{Map, Value};
use validator::Validate; use validator::Validate;
use vllm_engine_core_client::protocol::sampling::RepetitionDetectionParams;
use vllm_text::Prompt; use vllm_text::Prompt;
use crate::routes::openai::utils::types::{ use crate::routes::openai::utils::types::{
@@ -100,6 +101,9 @@ pub struct CompletionRequest {
/// Repetition penalty for reducing repetitive text /// Repetition penalty for reducing repetitive text
pub repetition_penalty: Option<f32>, pub repetition_penalty: Option<f32>,
/// Parameters for detecting repetitive N-gram patterns in output tokens
pub repetition_detection: Option<RepetitionDetectionParams>,
/// Length penalty for beam search /// Length penalty for beam search
pub length_penalty: Option<f32>, pub length_penalty: Option<f32>,
+3
View File
@@ -27,6 +27,8 @@ pub enum Error {
MinTokensExceedsMaxTokens { min_tokens: u32, max_tokens: u32 }, MinTokensExceedsMaxTokens { min_tokens: u32, max_tokens: u32 },
#[error("`thinking_token_budget` must be a non-negative integer or -1 for unlimited.")] #[error("`thinking_token_budget` must be a non-negative integer or -1 for unlimited.")]
InvalidThinkingTokenBudget, InvalidThinkingTokenBudget,
#[error("invalid repetition detection params: {message}")]
InvalidRepetitionDetection { message: String },
#[error("text request stream `{request_id}` closed before terminal output")] #[error("text request stream `{request_id}` closed before terminal output")]
StreamClosedBeforeTerminalOutput { request_id: String }, StreamClosedBeforeTerminalOutput { request_id: String },
#[error(transparent)] #[error(transparent)]
@@ -47,6 +49,7 @@ impl Error {
| Self::TokenIds(_) | Self::TokenIds(_)
| Self::MinTokensExceedsMaxTokens { .. } | Self::MinTokensExceedsMaxTokens { .. }
| Self::InvalidThinkingTokenBudget | Self::InvalidThinkingTokenBudget
| Self::InvalidRepetitionDetection { .. }
// An empty tokenized prompt detected later, at request prepare // An empty tokenized prompt detected later, at request prepare
// time, surfaces through the transparent Llm wrapper. // time, surfaces through the transparent Llm wrapper.
| Self::Llm(LlmError::EmptyPromptTokenIds { .. }) => true, | Self::Llm(LlmError::EmptyPromptTokenIds { .. }) => true,
+92 -1
View File
@@ -5,7 +5,9 @@ pub(crate) mod token_ids;
use logprobs::validate_logprobs; use logprobs::validate_logprobs;
use token_ids::{validate_prompt_token_ids, validate_vocab_range}; use token_ids::{validate_prompt_token_ids, validate_vocab_range};
use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams; use vllm_engine_core_client::protocol::sampling::{
EngineCoreSamplingParams, RepetitionDetectionParams,
};
use vllm_llm::GenerateRequest; use vllm_llm::GenerateRequest;
use vllm_tokenizer::Tokenizer; use vllm_tokenizer::Tokenizer;
@@ -93,6 +95,7 @@ pub fn lower_sampling_params(
frequency_penalty, frequency_penalty,
presence_penalty, presence_penalty,
repetition_penalty, repetition_penalty,
repetition_detection,
stop_token_ids, stop_token_ids,
ignore_eos, ignore_eos,
logit_bias, logit_bias,
@@ -110,6 +113,7 @@ pub fn lower_sampling_params(
logprob_token_ids.as_deref(), logprob_token_ids.as_deref(),
sampling_limits, sampling_limits,
)?; )?;
validate_repetition_detection(repetition_detection.as_ref())?;
// Mirrors the model-generation-config inheritance used by vLLM's OpenAI chat // Mirrors the model-generation-config inheritance used by vLLM's OpenAI chat
// path: https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/entrypoints/openai/chat_completion/protocol.py#L424-L450 // path: https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/entrypoints/openai/chat_completion/protocol.py#L424-L450
@@ -163,6 +167,7 @@ pub fn lower_sampling_params(
frequency_penalty, frequency_penalty,
presence_penalty, presence_penalty,
repetition_penalty, repetition_penalty,
repetition_detection: repetition_detection.filter(|p| !p.is_disabled()),
stop_token_ids, stop_token_ids,
eos_token_id: (!ignore_eos).then_some(primary_eos_token_id).flatten(), eos_token_id: (!ignore_eos).then_some(primary_eos_token_id).flatten(),
all_stop_token_ids, all_stop_token_ids,
@@ -194,6 +199,33 @@ fn normalize_thinking_token_budget(value: Option<i64>) -> Result<Option<u64>> {
} }
} }
fn validate_repetition_detection(params: Option<&RepetitionDetectionParams>) -> Result<()> {
let Some(params) = params else {
return Ok(());
};
if params.min_pattern_size > params.max_pattern_size {
return Err(Error::InvalidRepetitionDetection {
message: format!(
"`min_pattern_size` must be less than or equal to \
`max_pattern_size`, got min_pattern_size={}, \
max_pattern_size={}",
params.min_pattern_size, params.max_pattern_size
),
});
}
if params.max_pattern_size > 0 && params.min_count < 2 {
return Err(Error::InvalidRepetitionDetection {
message: format!(
"`min_count` must be at least 2, got min_count={}",
params.min_count
),
});
}
Ok(())
}
/// Convert bad-word strings into token-ID sequences, following the Python vLLM /// Convert bad-word strings into token-ID sequences, following the Python vLLM
/// logic in `SamplingParams.update_from_tokenizer()`. /// logic in `SamplingParams.update_from_tokenizer()`.
/// ///
@@ -390,6 +422,59 @@ mod tests {
)); ));
} }
#[test]
fn lower_sampling_params_validates_repetition_detection() {
let lower = |repetition_detection| {
lower_sampling_params_with_limits(
SamplingParams {
repetition_detection,
..SamplingParams::default()
},
sample_sampling_limits(),
)
};
let enabled = RepetitionDetectionParams {
max_pattern_size: 4,
min_pattern_size: 2,
min_count: 2,
};
assert_eq!(
lower(Some(enabled.clone())).unwrap().repetition_detection,
Some(enabled)
);
let disabled = RepetitionDetectionParams {
max_pattern_size: 0,
min_pattern_size: 0,
min_count: 0,
};
assert_eq!(lower(Some(disabled)).unwrap().repetition_detection, None);
let error = lower(Some(RepetitionDetectionParams {
max_pattern_size: 1,
min_pattern_size: 2,
min_count: 2,
}))
.unwrap_err();
let Error::InvalidRepetitionDetection { message } = error else {
panic!("expected repetition_detection validation error");
};
assert!(message.contains("min_pattern_size=2"));
assert!(message.contains("max_pattern_size=1"));
let error = lower(Some(RepetitionDetectionParams {
max_pattern_size: 1,
min_pattern_size: 1,
min_count: 1,
}))
.unwrap_err();
let Error::InvalidRepetitionDetection { message } = error else {
panic!("expected repetition_detection validation error");
};
assert!(message.contains("min_count=1"));
}
#[test] #[test]
fn lower_text_request_applies_python_style_eos_hints() { fn lower_text_request_applies_python_style_eos_hints() {
let prepared = lower_text_request( let prepared = lower_text_request(
@@ -417,6 +502,7 @@ mod tests {
frequency_penalty: 0.0, frequency_penalty: 0.0,
presence_penalty: 0.0, presence_penalty: 0.0,
repetition_penalty: 1.0, repetition_penalty: 1.0,
repetition_detection: None,
stop_token_ids: [ stop_token_ids: [
77, 77,
], ],
@@ -469,6 +555,7 @@ mod tests {
frequency_penalty: 0.0, frequency_penalty: 0.0,
presence_penalty: 0.0, presence_penalty: 0.0,
repetition_penalty: 1.0, repetition_penalty: 1.0,
repetition_detection: None,
stop_token_ids: [], stop_token_ids: [],
eos_token_id: None, eos_token_id: None,
all_stop_token_ids: { all_stop_token_ids: {
@@ -600,6 +687,7 @@ mod tests {
frequency_penalty: 0.0, frequency_penalty: 0.0,
presence_penalty: 0.0, presence_penalty: 0.0,
repetition_penalty: 1.0, repetition_penalty: 1.0,
repetition_detection: None,
stop_token_ids: [ stop_token_ids: [
151643, 151643,
], ],
@@ -662,6 +750,7 @@ mod tests {
frequency_penalty: 0.0, frequency_penalty: 0.0,
presence_penalty: 0.0, presence_penalty: 0.0,
repetition_penalty: 1.0, repetition_penalty: 1.0,
repetition_detection: None,
stop_token_ids: [ stop_token_ids: [
11, 11,
77, 77,
@@ -732,6 +821,7 @@ mod tests {
frequency_penalty: 0.0, frequency_penalty: 0.0,
presence_penalty: 0.0, presence_penalty: 0.0,
repetition_penalty: 1.2, repetition_penalty: 1.2,
repetition_detection: None,
stop_token_ids: [], stop_token_ids: [],
eos_token_id: None, eos_token_id: None,
all_stop_token_ids: {}, all_stop_token_ids: {},
@@ -980,6 +1070,7 @@ mod tests {
frequency_penalty: 0.0, frequency_penalty: 0.0,
presence_penalty: 0.0, presence_penalty: 0.0,
repetition_penalty: 1.2, repetition_penalty: 1.2,
repetition_detection: None,
stop_token_ids: [], stop_token_ids: [],
eos_token_id: None, eos_token_id: None,
all_stop_token_ids: {}, all_stop_token_ids: {},
+5
View File
@@ -6,6 +6,7 @@ use serde_json::Value;
use vllm_engine_core_client::protocol::lora::LoraRequest; use vllm_engine_core_client::protocol::lora::LoraRequest;
use vllm_engine_core_client::protocol::multimodal::MmFeatures; use vllm_engine_core_client::protocol::multimodal::MmFeatures;
use vllm_engine_core_client::protocol::request::ReasoningParserKwargs; use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
use vllm_engine_core_client::protocol::sampling::RepetitionDetectionParams;
use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsParams; use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsParams;
use crate::error::{Error, Result}; use crate::error::{Error, Result};
@@ -83,6 +84,9 @@ pub struct SamplingParams {
/// Repetition penalty applied by the sampler. `None` means no explicit user /// Repetition penalty applied by the sampler. `None` means no explicit user
/// override. /// override.
pub repetition_penalty: Option<f32>, pub repetition_penalty: Option<f32>,
/// Parameters for detecting repetitive N-gram patterns. `None` means no
/// explicit user override.
pub repetition_detection: Option<RepetitionDetectionParams>,
/// Explicit stop token IDs provided by the caller. `None` means no explicit /// Explicit stop token IDs provided by the caller. `None` means no explicit
/// user override. /// user override.
pub stop_token_ids: Option<Vec<u32>>, pub stop_token_ids: Option<Vec<u32>>,
@@ -130,6 +134,7 @@ impl Default for SamplingParams {
frequency_penalty: None, frequency_penalty: None,
presence_penalty: None, presence_penalty: None,
repetition_penalty: None, repetition_penalty: None,
repetition_detection: None,
stop_token_ids: None, stop_token_ids: None,
ignore_eos: false, ignore_eos: false,
logit_bias: None, logit_bias: None,