[Rust Frontend] add repetition_detection support to sampling params (#46684)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Bugen Zhao
parent
fbc9ba6d30
commit
6429d5f527
@@ -332,7 +332,7 @@ fn to_finish_info(finished: &Finished, token_ids: &[u32]) -> pb::FinishInfo {
|
||||
(PbFinishReason::Stop as i32, sr)
|
||||
}
|
||||
FinishReason::Length => (PbFinishReason::Length as i32, None),
|
||||
FinishReason::Abort | FinishReason::Error | FinishReason::Repetition => {
|
||||
FinishReason::Abort | FinishReason::Error | FinishReason::Repetition(_) => {
|
||||
(PbFinishReason::Aborted as i32, None)
|
||||
}
|
||||
};
|
||||
|
||||
@@ -805,7 +805,7 @@ fn chat_finish_reason_to_openai(
|
||||
FinishReason::Stop(_) => Ok("stop"),
|
||||
FinishReason::Length => Ok("length"),
|
||||
FinishReason::Abort => Ok("abort"),
|
||||
FinishReason::Repetition => Ok("stop"),
|
||||
FinishReason::Repetition(_) => Ok("repetition"),
|
||||
FinishReason::Error => {
|
||||
bail_server_error!("Internal server error");
|
||||
}
|
||||
|
||||
@@ -122,6 +122,7 @@ pub(super) fn prepare_chat_request(
|
||||
frequency_penalty: request.frequency_penalty,
|
||||
presence_penalty: request.presence_penalty,
|
||||
repetition_penalty: request.repetition_penalty,
|
||||
repetition_detection: request.repetition_detection,
|
||||
stop_token_ids: request.stop_token_ids,
|
||||
ignore_eos: request.ignore_eos,
|
||||
logit_bias: convert_logit_bias(request.logit_bias)?,
|
||||
|
||||
@@ -6,6 +6,7 @@ use serde_json::Value;
|
||||
use serde_with::SerializeDisplay;
|
||||
use validator::Validate;
|
||||
use vllm_chat::ReasoningEffort;
|
||||
use vllm_engine_core_client::protocol::sampling::RepetitionDetectionParams;
|
||||
|
||||
use crate::routes::openai::utils::structured_outputs::ResponseFormat;
|
||||
use crate::routes::openai::utils::types::{
|
||||
@@ -236,7 +237,7 @@ pub struct ChatCompletionRequest {
|
||||
pub vllm_xargs: Option<HashMap<String, Value>>,
|
||||
|
||||
/// Parameters for detecting repetitive N-gram patterns in output tokens
|
||||
pub repetition_detection: Option<Value>,
|
||||
pub repetition_detection: Option<RepetitionDetectionParams>,
|
||||
}
|
||||
|
||||
impl Default for ChatCompletionRequest {
|
||||
|
||||
@@ -94,11 +94,6 @@ pub(super) fn validate_request_compat(
|
||||
"mm_processor_kwargs",
|
||||
"mm_processor_kwargs is not supported.",
|
||||
)?;
|
||||
reject_non_default(
|
||||
request.repetition_detection.as_ref(),
|
||||
"repetition_detection",
|
||||
"repetition_detection is not supported.",
|
||||
)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -14,9 +14,11 @@ use axum::http::HeaderMap;
|
||||
use axum::response::sse::{Event, Sse};
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use futures::{Stream, StreamExt as _, pin_mut};
|
||||
use serde_json::Value;
|
||||
use thiserror_ext::AsReport as _;
|
||||
use tracing::{debug, error, info, trace};
|
||||
use tracing_futures::Instrument as _;
|
||||
use vllm_engine_core_client::protocol::output::StopReason;
|
||||
use vllm_text::{
|
||||
DecodedPromptLogprobs, DecodedTextEvent, FinishReason, TextOutputStream,
|
||||
TextOutputStreamExt as _,
|
||||
@@ -142,9 +144,7 @@ async fn collect_completion(
|
||||
.await
|
||||
.map_err(|error| server_error!("completion stream failed: {}", error.to_report_string()))?;
|
||||
let finish_reason = collected.finish_reason.clone();
|
||||
let stop_reason = finish_reason
|
||||
.as_stop_reason()
|
||||
.map(|sr| serde_json::to_value(sr).expect("StopReason must serialize to JSON"));
|
||||
let stop_reason = finish_reason.as_stop_reason().map(stop_reason_to_json);
|
||||
|
||||
let prompt_char_count = echo.as_ref().map(|prompt| text_len(prompt)).unwrap_or_default();
|
||||
let logprobs = if requested_logprobs.is_some() && prompt_only {
|
||||
@@ -181,7 +181,7 @@ async fn collect_completion(
|
||||
Some(prompt) if prompt_only => prompt.clone(),
|
||||
Some(prompt) => format!("{prompt}{}", collected.text),
|
||||
};
|
||||
let finish_reason = completion_finish_reason_to_openai(finish_reason)?.to_string();
|
||||
let finish_reason = completion_finish_reason_to_openai(&finish_reason)?.to_string();
|
||||
let usage = Usage::from_token_usage(collected.usage, enable_prompt_tokens_details);
|
||||
|
||||
if enable_log_requests {
|
||||
@@ -441,21 +441,24 @@ fn final_chunk(
|
||||
created: u64,
|
||||
finish_reason: FinishReason,
|
||||
) -> Result<CompletionStreamResponse, ApiError> {
|
||||
let finish_reason = completion_finish_reason_to_openai(finish_reason)?;
|
||||
let stop_reason = finish_reason.as_stop_reason().map(stop_reason_to_json);
|
||||
let finish_reason = completion_finish_reason_to_openai(&finish_reason)?;
|
||||
|
||||
let mut chunk = CompletionStreamResponse::new(request_id, response_model, created);
|
||||
chunk.choices.push(CompletionStreamChoice {
|
||||
finish_reason: Some(finish_reason.to_string()),
|
||||
stop_reason,
|
||||
..Default::default()
|
||||
});
|
||||
Ok(chunk)
|
||||
}
|
||||
|
||||
fn completion_finish_reason_to_openai(
|
||||
finish_reason: FinishReason,
|
||||
finish_reason: &FinishReason,
|
||||
) -> Result<&'static str, ApiError> {
|
||||
match finish_reason {
|
||||
FinishReason::Stop(_) | FinishReason::Repetition => Ok("stop"),
|
||||
FinishReason::Stop(_) => Ok("stop"),
|
||||
FinishReason::Repetition(_) => Ok("repetition"),
|
||||
FinishReason::Length => Ok("length"),
|
||||
FinishReason::Abort => Ok("abort"),
|
||||
FinishReason::Error => {
|
||||
@@ -464,6 +467,10 @@ fn completion_finish_reason_to_openai(
|
||||
}
|
||||
}
|
||||
|
||||
fn stop_reason_to_json(stop_reason: &StopReason) -> Value {
|
||||
serde_json::to_value(stop_reason).expect("StopReason must serialize to JSON")
|
||||
}
|
||||
|
||||
fn prompt_only_logprobs_to_openai(
|
||||
prompt_logprobs: Option<&DecodedPromptLogprobs>,
|
||||
prompt: &str,
|
||||
@@ -577,6 +584,7 @@ fn done_sse_event() -> Event {
|
||||
mod tests {
|
||||
use futures::{StreamExt as _, stream};
|
||||
use itertools::Itertools as _;
|
||||
use vllm_engine_core_client::protocol::output::StopReason;
|
||||
use vllm_text::{
|
||||
DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedTextEvent,
|
||||
DecodedTokenLogprob, FinishReason, Finished,
|
||||
@@ -670,7 +678,9 @@ mod tests {
|
||||
output_token_count: 2,
|
||||
cached_token_count: 3,
|
||||
},
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
finish_reason: FinishReason::Repetition(Some(StopReason::Text(
|
||||
"repetition_detected".to_string(),
|
||||
))),
|
||||
kv_transfer_params: None,
|
||||
}),
|
||||
}),
|
||||
@@ -726,6 +736,20 @@ mod tests {
|
||||
CompletionSseChunk::Usage(_) => panic!("expected regular chunk"),
|
||||
}
|
||||
|
||||
match &chunks[2] {
|
||||
CompletionSseChunk::Chunk(chunk) => {
|
||||
assert_eq!(
|
||||
chunk.choices[0].finish_reason.as_deref(),
|
||||
Some("repetition")
|
||||
);
|
||||
assert_eq!(
|
||||
chunk.choices[0].stop_reason,
|
||||
Some(serde_json::json!("repetition_detected"))
|
||||
);
|
||||
}
|
||||
CompletionSseChunk::Usage(_) => panic!("expected regular chunk"),
|
||||
}
|
||||
|
||||
match &chunks[3] {
|
||||
CompletionSseChunk::Usage(chunk) => {
|
||||
assert_eq!(
|
||||
|
||||
@@ -117,6 +117,7 @@ pub(super) fn prepare_completion_request(
|
||||
frequency_penalty: request.frequency_penalty,
|
||||
presence_penalty: request.presence_penalty,
|
||||
repetition_penalty: request.repetition_penalty,
|
||||
repetition_detection: request.repetition_detection,
|
||||
stop_token_ids: request.stop_token_ids,
|
||||
ignore_eos: request.ignore_eos,
|
||||
logit_bias: convert_logit_bias(request.logit_bias)?,
|
||||
|
||||
@@ -3,6 +3,7 @@ use std::collections::HashMap;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
use validator::Validate;
|
||||
use vllm_engine_core_client::protocol::sampling::RepetitionDetectionParams;
|
||||
use vllm_text::Prompt;
|
||||
|
||||
use crate::routes::openai::utils::types::{
|
||||
@@ -100,6 +101,9 @@ pub struct CompletionRequest {
|
||||
/// Repetition penalty for reducing repetitive text
|
||||
pub repetition_penalty: Option<f32>,
|
||||
|
||||
/// Parameters for detecting repetitive N-gram patterns in output tokens
|
||||
pub repetition_detection: Option<RepetitionDetectionParams>,
|
||||
|
||||
/// Length penalty for beam search
|
||||
pub length_penalty: Option<f32>,
|
||||
|
||||
|
||||
Reference in New Issue
Block a user