[Bugfix][Rust Frontend] Limit chat top_logprobs in responses (#48134)
Signed-off-by: reidliu41 <reid201711@gmail.com>
This commit is contained in:
@@ -125,6 +125,7 @@ async fn collect_chat_completion(
|
||||
// Ignored: non-streaming responses are collected before usage is attached.
|
||||
include_continuous_usage: _,
|
||||
requested_logprobs,
|
||||
output_top_logprobs,
|
||||
include_prompt_logprobs,
|
||||
include_reasoning,
|
||||
echo,
|
||||
@@ -173,6 +174,7 @@ async fn collect_chat_completion(
|
||||
logprobs.as_ref().ok_or_else(|| {
|
||||
server_error!("chat response requested logprobs but generation returned none")
|
||||
})?,
|
||||
output_top_logprobs,
|
||||
return_tokens_as_token_ids,
|
||||
)?)
|
||||
} else {
|
||||
@@ -248,6 +250,7 @@ async fn chat_completion_chunk_stream(
|
||||
include_usage,
|
||||
include_continuous_usage,
|
||||
requested_logprobs,
|
||||
output_top_logprobs,
|
||||
// Ignored: chat streaming prompt logprobs are rejected for Python parity.
|
||||
include_prompt_logprobs: _,
|
||||
include_reasoning,
|
||||
@@ -337,7 +340,13 @@ async fn chat_completion_chunk_stream(
|
||||
let openai_logprobs = if include_metadata {
|
||||
logprobs
|
||||
.as_ref()
|
||||
.map(|lp| decoded_logprobs_to_openai_chat(lp, return_tokens_as_token_ids))
|
||||
.map(|lp| {
|
||||
decoded_logprobs_to_openai_chat(
|
||||
lp,
|
||||
output_top_logprobs,
|
||||
return_tokens_as_token_ids,
|
||||
)
|
||||
})
|
||||
.transpose()?
|
||||
} else {
|
||||
None
|
||||
|
||||
@@ -42,6 +42,8 @@ pub(super) struct ResponseOptions {
|
||||
pub include_continuous_usage: bool,
|
||||
/// Whether the caller requested output logprobs on chat choices.
|
||||
pub requested_logprobs: bool,
|
||||
/// Number of top logprobs to include for each output token.
|
||||
pub output_top_logprobs: i32,
|
||||
/// Whether the caller requested top-level prompt logprobs.
|
||||
pub include_prompt_logprobs: bool,
|
||||
/// Whether to include reasoning content in OpenAI responses.
|
||||
@@ -172,6 +174,7 @@ pub(super) fn prepare_chat_request(
|
||||
include_usage,
|
||||
include_continuous_usage,
|
||||
requested_logprobs,
|
||||
output_top_logprobs: top_logprobs,
|
||||
include_prompt_logprobs,
|
||||
include_reasoning,
|
||||
echo,
|
||||
|
||||
@@ -120,12 +120,13 @@ pub fn decoded_prompt_logprobs_to_maps(
|
||||
/// shape.
|
||||
pub fn decoded_logprobs_to_openai_chat(
|
||||
logprobs: &DecodedLogprobs,
|
||||
top_logprobs: i32,
|
||||
return_tokens_as_token_ids: bool,
|
||||
) -> Result<ChatLogProbs, ApiError> {
|
||||
let content = logprobs
|
||||
.positions
|
||||
.iter()
|
||||
.map(|pos| position_to_chat_logprobs_content(pos, return_tokens_as_token_ids))
|
||||
.map(|pos| position_to_chat_logprobs_content(pos, top_logprobs, return_tokens_as_token_ids))
|
||||
.try_collect()?;
|
||||
|
||||
Ok(ChatLogProbs {
|
||||
@@ -224,6 +225,7 @@ fn position_top_logprobs_map(
|
||||
|
||||
fn position_to_chat_logprobs_content(
|
||||
position: &DecodedPositionLogprobs,
|
||||
top_logprobs: i32,
|
||||
return_tokens_as_token_ids: bool,
|
||||
) -> Result<ChatLogProbsContent, ApiError> {
|
||||
let chosen = position.entries.first().ok_or_else(|| {
|
||||
@@ -235,9 +237,7 @@ fn position_to_chat_logprobs_content(
|
||||
token: token_str.clone(),
|
||||
logprob: clamp_logprob(chosen.logprob),
|
||||
bytes: Some(token_bytes(&token_str)),
|
||||
top_logprobs: position
|
||||
.entries
|
||||
.iter()
|
||||
top_logprobs: chat_top_logprob_entries(position, top_logprobs)
|
||||
.map(|entry| {
|
||||
let t = format_token(entry, return_tokens_as_token_ids);
|
||||
TopLogProb {
|
||||
@@ -250,6 +250,19 @@ fn position_to_chat_logprobs_content(
|
||||
})
|
||||
}
|
||||
|
||||
fn chat_top_logprob_entries(
|
||||
position: &DecodedPositionLogprobs,
|
||||
top_logprobs: i32,
|
||||
) -> impl Iterator<Item = &DecodedTokenLogprob> {
|
||||
let limit = if top_logprobs == -1 {
|
||||
position.entries.len()
|
||||
} else {
|
||||
usize::try_from(top_logprobs).unwrap_or(0)
|
||||
};
|
||||
|
||||
position.entries.iter().take(limit)
|
||||
}
|
||||
|
||||
fn token_bytes(token: &str) -> Vec<u8> {
|
||||
token.as_bytes().to_vec()
|
||||
}
|
||||
@@ -257,3 +270,51 @@ fn token_bytes(token: &str) -> Vec<u8> {
|
||||
pub fn clamp_logprob(logprob: f32) -> f32 {
|
||||
logprob.max(-9999.0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use vllm_text::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTokenLogprob};
|
||||
|
||||
use super::decoded_logprobs_to_openai_chat;
|
||||
|
||||
fn sample_logprobs() -> DecodedLogprobs {
|
||||
DecodedLogprobs {
|
||||
positions: vec![DecodedPositionLogprobs {
|
||||
entries: vec![
|
||||
DecodedTokenLogprob {
|
||||
token_id: 1,
|
||||
token: "A".to_string(),
|
||||
logprob: -0.1,
|
||||
rank: 1,
|
||||
},
|
||||
DecodedTokenLogprob {
|
||||
token_id: 2,
|
||||
token: "B".to_string(),
|
||||
logprob: -1.0,
|
||||
rank: 2,
|
||||
},
|
||||
DecodedTokenLogprob {
|
||||
token_id: 3,
|
||||
token: "C".to_string(),
|
||||
logprob: -2.0,
|
||||
rank: 3,
|
||||
},
|
||||
],
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
fn chat_top_logprobs_len(top_logprobs: i32) -> usize {
|
||||
let chat_logprobs =
|
||||
decoded_logprobs_to_openai_chat(&sample_logprobs(), top_logprobs, false)
|
||||
.expect("chat logprobs");
|
||||
chat_logprobs.content.expect("content")[0].top_logprobs.len()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_logprobs_respects_requested_top_logprobs_count() {
|
||||
assert_eq!(chat_top_logprobs_len(0), 0);
|
||||
assert_eq!(chat_top_logprobs_len(1), 1);
|
||||
assert_eq!(chat_top_logprobs_len(-1), 3);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2380,6 +2380,14 @@ async fn non_stream_chat_includes_logprobs_and_prompt_logprobs() {
|
||||
json["choices"][0]["logprobs"]["content"][1]["token"],
|
||||
json!("i")
|
||||
);
|
||||
assert_eq!(
|
||||
json["choices"][0]["logprobs"]["content"][0]["top_logprobs"],
|
||||
json!([])
|
||||
);
|
||||
assert_eq!(
|
||||
json["choices"][0]["logprobs"]["content"][1]["top_logprobs"],
|
||||
json!([])
|
||||
);
|
||||
assert_eq!(json["prompt_logprobs"][0], serde_json::Value::Null);
|
||||
assert!(json["prompt_logprobs"][1].is_object());
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user