[Rust Frontend] add repetition_detection support to sampling params (#46684)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
yangyang-cs95
2026-07-03 14:06:23 +08:00
committed by GitHub
co-authored by Bugen Zhao
parent fbc9ba6d30
commit 6429d5f527
14 changed files with 177 additions and 20 deletions
+1 -1
View File
@@ -332,7 +332,7 @@ fn to_finish_info(finished: &Finished, token_ids: &[u32]) -> pb::FinishInfo {
(PbFinishReason::Stop as i32, sr)
}
FinishReason::Length => (PbFinishReason::Length as i32, None),
FinishReason::Abort | FinishReason::Error | FinishReason::Repetition => {
FinishReason::Abort | FinishReason::Error | FinishReason::Repetition(_) => {
(PbFinishReason::Aborted as i32, None)
}
};
@@ -805,7 +805,7 @@ fn chat_finish_reason_to_openai(
FinishReason::Stop(_) => Ok("stop"),
FinishReason::Length => Ok("length"),
FinishReason::Abort => Ok("abort"),
FinishReason::Repetition => Ok("stop"),
FinishReason::Repetition(_) => Ok("repetition"),
FinishReason::Error => {
bail_server_error!("Internal server error");
}
@@ -122,6 +122,7 @@ pub(super) fn prepare_chat_request(
frequency_penalty: request.frequency_penalty,
presence_penalty: request.presence_penalty,
repetition_penalty: request.repetition_penalty,
repetition_detection: request.repetition_detection,
stop_token_ids: request.stop_token_ids,
ignore_eos: request.ignore_eos,
logit_bias: convert_logit_bias(request.logit_bias)?,
@@ -6,6 +6,7 @@ use serde_json::Value;
use serde_with::SerializeDisplay;
use validator::Validate;
use vllm_chat::ReasoningEffort;
use vllm_engine_core_client::protocol::sampling::RepetitionDetectionParams;
use crate::routes::openai::utils::structured_outputs::ResponseFormat;
use crate::routes::openai::utils::types::{
@@ -236,7 +237,7 @@ pub struct ChatCompletionRequest {
pub vllm_xargs: Option<HashMap<String, Value>>,
/// Parameters for detecting repetitive N-gram patterns in output tokens
pub repetition_detection: Option<Value>,
pub repetition_detection: Option<RepetitionDetectionParams>,
}
impl Default for ChatCompletionRequest {
@@ -94,11 +94,6 @@ pub(super) fn validate_request_compat(
"mm_processor_kwargs",
"mm_processor_kwargs is not supported.",
)?;
reject_non_default(
request.repetition_detection.as_ref(),
"repetition_detection",
"repetition_detection is not supported.",
)?;
Ok(())
}
@@ -14,9 +14,11 @@ use axum::http::HeaderMap;
use axum::response::sse::{Event, Sse};
use axum::response::{IntoResponse, Response};
use futures::{Stream, StreamExt as _, pin_mut};
use serde_json::Value;
use thiserror_ext::AsReport as _;
use tracing::{debug, error, info, trace};
use tracing_futures::Instrument as _;
use vllm_engine_core_client::protocol::output::StopReason;
use vllm_text::{
DecodedPromptLogprobs, DecodedTextEvent, FinishReason, TextOutputStream,
TextOutputStreamExt as _,
@@ -142,9 +144,7 @@ async fn collect_completion(
.await
.map_err(|error| server_error!("completion stream failed: {}", error.to_report_string()))?;
let finish_reason = collected.finish_reason.clone();
let stop_reason = finish_reason
.as_stop_reason()
.map(|sr| serde_json::to_value(sr).expect("StopReason must serialize to JSON"));
let stop_reason = finish_reason.as_stop_reason().map(stop_reason_to_json);
let prompt_char_count = echo.as_ref().map(|prompt| text_len(prompt)).unwrap_or_default();
let logprobs = if requested_logprobs.is_some() && prompt_only {
@@ -181,7 +181,7 @@ async fn collect_completion(
Some(prompt) if prompt_only => prompt.clone(),
Some(prompt) => format!("{prompt}{}", collected.text),
};
let finish_reason = completion_finish_reason_to_openai(finish_reason)?.to_string();
let finish_reason = completion_finish_reason_to_openai(&finish_reason)?.to_string();
let usage = Usage::from_token_usage(collected.usage, enable_prompt_tokens_details);
if enable_log_requests {
@@ -441,21 +441,24 @@ fn final_chunk(
created: u64,
finish_reason: FinishReason,
) -> Result<CompletionStreamResponse, ApiError> {
let finish_reason = completion_finish_reason_to_openai(finish_reason)?;
let stop_reason = finish_reason.as_stop_reason().map(stop_reason_to_json);
let finish_reason = completion_finish_reason_to_openai(&finish_reason)?;
let mut chunk = CompletionStreamResponse::new(request_id, response_model, created);
chunk.choices.push(CompletionStreamChoice {
finish_reason: Some(finish_reason.to_string()),
stop_reason,
..Default::default()
});
Ok(chunk)
}
fn completion_finish_reason_to_openai(
finish_reason: FinishReason,
finish_reason: &FinishReason,
) -> Result<&'static str, ApiError> {
match finish_reason {
FinishReason::Stop(_) | FinishReason::Repetition => Ok("stop"),
FinishReason::Stop(_) => Ok("stop"),
FinishReason::Repetition(_) => Ok("repetition"),
FinishReason::Length => Ok("length"),
FinishReason::Abort => Ok("abort"),
FinishReason::Error => {
@@ -464,6 +467,10 @@ fn completion_finish_reason_to_openai(
}
}
fn stop_reason_to_json(stop_reason: &StopReason) -> Value {
serde_json::to_value(stop_reason).expect("StopReason must serialize to JSON")
}
fn prompt_only_logprobs_to_openai(
prompt_logprobs: Option<&DecodedPromptLogprobs>,
prompt: &str,
@@ -577,6 +584,7 @@ fn done_sse_event() -> Event {
mod tests {
use futures::{StreamExt as _, stream};
use itertools::Itertools as _;
use vllm_engine_core_client::protocol::output::StopReason;
use vllm_text::{
DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedTextEvent,
DecodedTokenLogprob, FinishReason, Finished,
@@ -670,7 +678,9 @@ mod tests {
output_token_count: 2,
cached_token_count: 3,
},
finish_reason: FinishReason::stop_eos(),
finish_reason: FinishReason::Repetition(Some(StopReason::Text(
"repetition_detected".to_string(),
))),
kv_transfer_params: None,
}),
}),
@@ -726,6 +736,20 @@ mod tests {
CompletionSseChunk::Usage(_) => panic!("expected regular chunk"),
}
match &chunks[2] {
CompletionSseChunk::Chunk(chunk) => {
assert_eq!(
chunk.choices[0].finish_reason.as_deref(),
Some("repetition")
);
assert_eq!(
chunk.choices[0].stop_reason,
Some(serde_json::json!("repetition_detected"))
);
}
CompletionSseChunk::Usage(_) => panic!("expected regular chunk"),
}
match &chunks[3] {
CompletionSseChunk::Usage(chunk) => {
assert_eq!(
@@ -117,6 +117,7 @@ pub(super) fn prepare_completion_request(
frequency_penalty: request.frequency_penalty,
presence_penalty: request.presence_penalty,
repetition_penalty: request.repetition_penalty,
repetition_detection: request.repetition_detection,
stop_token_ids: request.stop_token_ids,
ignore_eos: request.ignore_eos,
logit_bias: convert_logit_bias(request.logit_bias)?,
@@ -3,6 +3,7 @@ use std::collections::HashMap;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use validator::Validate;
use vllm_engine_core_client::protocol::sampling::RepetitionDetectionParams;
use vllm_text::Prompt;
use crate::routes::openai::utils::types::{
@@ -100,6 +101,9 @@ pub struct CompletionRequest {
/// Repetition penalty for reducing repetitive text
pub repetition_penalty: Option<f32>,
/// Parameters for detecting repetitive N-gram patterns in output tokens
pub repetition_detection: Option<RepetitionDetectionParams>,
/// Length penalty for beam search
pub length_penalty: Option<f32>,