diff --git a/rust/src/server/src/routes/openai/chat_completions.rs b/rust/src/server/src/routes/openai/chat_completions.rs index 41ef192b73..b08c533a51 100644 --- a/rust/src/server/src/routes/openai/chat_completions.rs +++ b/rust/src/server/src/routes/openai/chat_completions.rs @@ -125,6 +125,7 @@ async fn collect_chat_completion( // Ignored: non-streaming responses are collected before usage is attached. include_continuous_usage: _, requested_logprobs, + output_top_logprobs, include_prompt_logprobs, include_reasoning, echo, @@ -173,6 +174,7 @@ async fn collect_chat_completion( logprobs.as_ref().ok_or_else(|| { server_error!("chat response requested logprobs but generation returned none") })?, + output_top_logprobs, return_tokens_as_token_ids, )?) } else { @@ -248,6 +250,7 @@ async fn chat_completion_chunk_stream( include_usage, include_continuous_usage, requested_logprobs, + output_top_logprobs, // Ignored: chat streaming prompt logprobs are rejected for Python parity. include_prompt_logprobs: _, include_reasoning, @@ -337,7 +340,13 @@ async fn chat_completion_chunk_stream( let openai_logprobs = if include_metadata { logprobs .as_ref() - .map(|lp| decoded_logprobs_to_openai_chat(lp, return_tokens_as_token_ids)) + .map(|lp| { + decoded_logprobs_to_openai_chat( + lp, + output_top_logprobs, + return_tokens_as_token_ids, + ) + }) .transpose()? } else { None diff --git a/rust/src/server/src/routes/openai/chat_completions/convert.rs b/rust/src/server/src/routes/openai/chat_completions/convert.rs index cfb4b1a767..5b2fa3c19e 100644 --- a/rust/src/server/src/routes/openai/chat_completions/convert.rs +++ b/rust/src/server/src/routes/openai/chat_completions/convert.rs @@ -42,6 +42,8 @@ pub(super) struct ResponseOptions { pub include_continuous_usage: bool, /// Whether the caller requested output logprobs on chat choices. pub requested_logprobs: bool, + /// Number of top logprobs to include for each output token. + pub output_top_logprobs: i32, /// Whether the caller requested top-level prompt logprobs. pub include_prompt_logprobs: bool, /// Whether to include reasoning content in OpenAI responses. @@ -172,6 +174,7 @@ pub(super) fn prepare_chat_request( include_usage, include_continuous_usage, requested_logprobs, + output_top_logprobs: top_logprobs, include_prompt_logprobs, include_reasoning, echo, diff --git a/rust/src/server/src/routes/openai/utils/logprobs.rs b/rust/src/server/src/routes/openai/utils/logprobs.rs index 19215755f5..078c42b507 100644 --- a/rust/src/server/src/routes/openai/utils/logprobs.rs +++ b/rust/src/server/src/routes/openai/utils/logprobs.rs @@ -120,12 +120,13 @@ pub fn decoded_prompt_logprobs_to_maps( /// shape. pub fn decoded_logprobs_to_openai_chat( logprobs: &DecodedLogprobs, + top_logprobs: i32, return_tokens_as_token_ids: bool, ) -> Result { let content = logprobs .positions .iter() - .map(|pos| position_to_chat_logprobs_content(pos, return_tokens_as_token_ids)) + .map(|pos| position_to_chat_logprobs_content(pos, top_logprobs, return_tokens_as_token_ids)) .try_collect()?; Ok(ChatLogProbs { @@ -224,6 +225,7 @@ fn position_top_logprobs_map( fn position_to_chat_logprobs_content( position: &DecodedPositionLogprobs, + top_logprobs: i32, return_tokens_as_token_ids: bool, ) -> Result { let chosen = position.entries.first().ok_or_else(|| { @@ -235,9 +237,7 @@ fn position_to_chat_logprobs_content( token: token_str.clone(), logprob: clamp_logprob(chosen.logprob), bytes: Some(token_bytes(&token_str)), - top_logprobs: position - .entries - .iter() + top_logprobs: chat_top_logprob_entries(position, top_logprobs) .map(|entry| { let t = format_token(entry, return_tokens_as_token_ids); TopLogProb { @@ -250,6 +250,19 @@ fn position_to_chat_logprobs_content( }) } +fn chat_top_logprob_entries( + position: &DecodedPositionLogprobs, + top_logprobs: i32, +) -> impl Iterator { + let limit = if top_logprobs == -1 { + position.entries.len() + } else { + usize::try_from(top_logprobs).unwrap_or(0) + }; + + position.entries.iter().take(limit) +} + fn token_bytes(token: &str) -> Vec { token.as_bytes().to_vec() } @@ -257,3 +270,51 @@ fn token_bytes(token: &str) -> Vec { pub fn clamp_logprob(logprob: f32) -> f32 { logprob.max(-9999.0) } + +#[cfg(test)] +mod tests { + use vllm_text::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTokenLogprob}; + + use super::decoded_logprobs_to_openai_chat; + + fn sample_logprobs() -> DecodedLogprobs { + DecodedLogprobs { + positions: vec![DecodedPositionLogprobs { + entries: vec![ + DecodedTokenLogprob { + token_id: 1, + token: "A".to_string(), + logprob: -0.1, + rank: 1, + }, + DecodedTokenLogprob { + token_id: 2, + token: "B".to_string(), + logprob: -1.0, + rank: 2, + }, + DecodedTokenLogprob { + token_id: 3, + token: "C".to_string(), + logprob: -2.0, + rank: 3, + }, + ], + }], + } + } + + fn chat_top_logprobs_len(top_logprobs: i32) -> usize { + let chat_logprobs = + decoded_logprobs_to_openai_chat(&sample_logprobs(), top_logprobs, false) + .expect("chat logprobs"); + chat_logprobs.content.expect("content")[0].top_logprobs.len() + } + + #[test] + fn chat_logprobs_respects_requested_top_logprobs_count() { + assert_eq!(chat_top_logprobs_len(0), 0); + assert_eq!(chat_top_logprobs_len(1), 1); + assert_eq!(chat_top_logprobs_len(-1), 3); + } +} diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index d72226c2fc..64e904b562 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -2380,6 +2380,14 @@ async fn non_stream_chat_includes_logprobs_and_prompt_logprobs() { json["choices"][0]["logprobs"]["content"][1]["token"], json!("i") ); + assert_eq!( + json["choices"][0]["logprobs"]["content"][0]["top_logprobs"], + json!([]) + ); + assert_eq!( + json["choices"][0]["logprobs"]["content"][1]["top_logprobs"], + json!([]) + ); assert_eq!(json["prompt_logprobs"][0], serde_json::Value::Null); assert!(json["prompt_logprobs"][1].is_object()); }