[Bugfix][Rust Frontend] Limit chat top_logprobs in responses (#48134)

Signed-off-by: reidliu41 <reid201711@gmail.com>
This commit is contained in:
Reid
2026-07-16 09:41:44 +00:00
committed by GitHub
parent 9f8cbfd8eb
commit 8c3393f373
4 changed files with 86 additions and 5 deletions
@@ -125,6 +125,7 @@ async fn collect_chat_completion(
// Ignored: non-streaming responses are collected before usage is attached.
include_continuous_usage: _,
requested_logprobs,
output_top_logprobs,
include_prompt_logprobs,
include_reasoning,
echo,
@@ -173,6 +174,7 @@ async fn collect_chat_completion(
logprobs.as_ref().ok_or_else(|| {
server_error!("chat response requested logprobs but generation returned none")
})?,
output_top_logprobs,
return_tokens_as_token_ids,
)?)
} else {
@@ -248,6 +250,7 @@ async fn chat_completion_chunk_stream(
include_usage,
include_continuous_usage,
requested_logprobs,
output_top_logprobs,
// Ignored: chat streaming prompt logprobs are rejected for Python parity.
include_prompt_logprobs: _,
include_reasoning,
@@ -337,7 +340,13 @@ async fn chat_completion_chunk_stream(
let openai_logprobs = if include_metadata {
logprobs
.as_ref()
.map(|lp| decoded_logprobs_to_openai_chat(lp, return_tokens_as_token_ids))
.map(|lp| {
decoded_logprobs_to_openai_chat(
lp,
output_top_logprobs,
return_tokens_as_token_ids,
)
})
.transpose()?
} else {
None
@@ -42,6 +42,8 @@ pub(super) struct ResponseOptions {
pub include_continuous_usage: bool,
/// Whether the caller requested output logprobs on chat choices.
pub requested_logprobs: bool,
/// Number of top logprobs to include for each output token.
pub output_top_logprobs: i32,
/// Whether the caller requested top-level prompt logprobs.
pub include_prompt_logprobs: bool,
/// Whether to include reasoning content in OpenAI responses.
@@ -172,6 +174,7 @@ pub(super) fn prepare_chat_request(
include_usage,
include_continuous_usage,
requested_logprobs,
output_top_logprobs: top_logprobs,
include_prompt_logprobs,
include_reasoning,
echo,
@@ -120,12 +120,13 @@ pub fn decoded_prompt_logprobs_to_maps(
/// shape.
pub fn decoded_logprobs_to_openai_chat(
logprobs: &DecodedLogprobs,
top_logprobs: i32,
return_tokens_as_token_ids: bool,
) -> Result<ChatLogProbs, ApiError> {
let content = logprobs
.positions
.iter()
.map(|pos| position_to_chat_logprobs_content(pos, return_tokens_as_token_ids))
.map(|pos| position_to_chat_logprobs_content(pos, top_logprobs, return_tokens_as_token_ids))
.try_collect()?;
Ok(ChatLogProbs {
@@ -224,6 +225,7 @@ fn position_top_logprobs_map(
fn position_to_chat_logprobs_content(
position: &DecodedPositionLogprobs,
top_logprobs: i32,
return_tokens_as_token_ids: bool,
) -> Result<ChatLogProbsContent, ApiError> {
let chosen = position.entries.first().ok_or_else(|| {
@@ -235,9 +237,7 @@ fn position_to_chat_logprobs_content(
token: token_str.clone(),
logprob: clamp_logprob(chosen.logprob),
bytes: Some(token_bytes(&token_str)),
top_logprobs: position
.entries
.iter()
top_logprobs: chat_top_logprob_entries(position, top_logprobs)
.map(|entry| {
let t = format_token(entry, return_tokens_as_token_ids);
TopLogProb {
@@ -250,6 +250,19 @@ fn position_to_chat_logprobs_content(
})
}
fn chat_top_logprob_entries(
position: &DecodedPositionLogprobs,
top_logprobs: i32,
) -> impl Iterator<Item = &DecodedTokenLogprob> {
let limit = if top_logprobs == -1 {
position.entries.len()
} else {
usize::try_from(top_logprobs).unwrap_or(0)
};
position.entries.iter().take(limit)
}
fn token_bytes(token: &str) -> Vec<u8> {
token.as_bytes().to_vec()
}
@@ -257,3 +270,51 @@ fn token_bytes(token: &str) -> Vec<u8> {
pub fn clamp_logprob(logprob: f32) -> f32 {
logprob.max(-9999.0)
}
#[cfg(test)]
mod tests {
use vllm_text::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTokenLogprob};
use super::decoded_logprobs_to_openai_chat;
fn sample_logprobs() -> DecodedLogprobs {
DecodedLogprobs {
positions: vec![DecodedPositionLogprobs {
entries: vec![
DecodedTokenLogprob {
token_id: 1,
token: "A".to_string(),
logprob: -0.1,
rank: 1,
},
DecodedTokenLogprob {
token_id: 2,
token: "B".to_string(),
logprob: -1.0,
rank: 2,
},
DecodedTokenLogprob {
token_id: 3,
token: "C".to_string(),
logprob: -2.0,
rank: 3,
},
],
}],
}
}
fn chat_top_logprobs_len(top_logprobs: i32) -> usize {
let chat_logprobs =
decoded_logprobs_to_openai_chat(&sample_logprobs(), top_logprobs, false)
.expect("chat logprobs");
chat_logprobs.content.expect("content")[0].top_logprobs.len()
}
#[test]
fn chat_logprobs_respects_requested_top_logprobs_count() {
assert_eq!(chat_top_logprobs_len(0), 0);
assert_eq!(chat_top_logprobs_len(1), 1);
assert_eq!(chat_top_logprobs_len(-1), 3);
}
}
+8
View File
@@ -2380,6 +2380,14 @@ async fn non_stream_chat_includes_logprobs_and_prompt_logprobs() {
json["choices"][0]["logprobs"]["content"][1]["token"],
json!("i")
);
assert_eq!(
json["choices"][0]["logprobs"]["content"][0]["top_logprobs"],
json!([])
);
assert_eq!(
json["choices"][0]["logprobs"]["content"][1]["top_logprobs"],
json!([])
);
assert_eq!(json["prompt_logprobs"][0], serde_json::Value::Null);
assert!(json["prompt_logprobs"][1].is_object());
}