diff --git a/rust/src/server/src/routes/inference/generate.rs b/rust/src/server/src/routes/inference/generate.rs index 6fcdb427cc..e176938db4 100644 --- a/rust/src/server/src/routes/inference/generate.rs +++ b/rust/src/server/src/routes/inference/generate.rs @@ -238,13 +238,18 @@ fn collect_generate( None }; let prompt_logprobs = if include_prompt_logprobs { - let prompt_logprobs = collected.prompt_logprobs.as_ref().ok_or_else(|| { - ApiError::server_error( - "raw generate response requested prompt_logprobs but generation returned none" - .to_string(), - ) - })?; - Some(raw_prompt_logprobs_to_maps(prompt_logprobs)) + match collected.prompt_logprobs.as_ref() { + Some(prompt_logprobs) => Some(raw_prompt_logprobs_to_maps(prompt_logprobs)), + // A single-token prompt has no scored positions; same mapping + // as /v1/completions. + None if collected.prompt_token_ids.len() == 1 => Some(vec![None]), + None => { + return Err(ApiError::server_error( + "raw generate response requested prompt_logprobs but generation returned none" + .to_string(), + )); + } + } } else { None }; @@ -472,4 +477,48 @@ mod tests { Some(2) ); } + + #[test] + fn collect_generate_maps_prompt_logprobs_for_single_token_prompt() { + let output_without_payload = |prompt_token_ids: Vec| CollectedGenerateOutput { + request_id: "raw-1".to_string(), + prompt_logprobs: None, + token_ids: vec![3], + logprobs: None, + finish_reason: FinishReason::stop_eos(), + usage: vllm_llm::TokenUsage { + prompt_token_count: prompt_token_ids.len(), + output_token_count: 1, + cached_token_count: 0, + }, + kv_transfer_params: None, + ec_transfer_params: None, + prompt_token_ids, + }; + + let response = collect_generate( + output_without_payload(vec![9707]), + "raw-1".to_string(), + ApiServerOptions::default(), + ResponseOptions { + include_prompt_logprobs: true, + ..Default::default() + }, + ) + .expect("single-token prompt without payload maps to [None]"); + let prompt_logprobs = response.prompt_logprobs.expect("prompt logprobs present"); + assert_eq!(prompt_logprobs.len(), 1); + assert!(prompt_logprobs[0].is_none()); + + collect_generate( + output_without_payload(vec![9707, 11]), + "raw-2".to_string(), + ApiServerOptions::default(), + ResponseOptions { + include_prompt_logprobs: true, + ..Default::default() + }, + ) + .expect_err("multi-token prompt without payload is an engine failure"); + } } diff --git a/rust/src/server/src/routes/openai/chat_completions.rs b/rust/src/server/src/routes/openai/chat_completions.rs index b08c533a51..bd86a06136 100644 --- a/rust/src/server/src/routes/openai/chat_completions.rs +++ b/rust/src/server/src/routes/openai/chat_completions.rs @@ -35,7 +35,7 @@ use crate::routes::openai::chat_completions::types::{ ChatMessageDelta, }; use crate::routes::openai::utils::logprobs::{ - decoded_logprobs_to_openai_chat, decoded_prompt_logprobs_to_maps, + decoded_logprobs_to_openai_chat, prompt_logprobs_to_maps, }; use crate::routes::openai::utils::types::{ ChatLogProbs, FunctionCallDelta, FunctionCallResponse, ToolCall, ToolCallDelta, Usage, @@ -181,14 +181,11 @@ async fn collect_chat_completion( None }; let prompt_logprobs = if include_prompt_logprobs { - Some(decoded_prompt_logprobs_to_maps( - prompt_logprobs.as_ref().ok_or_else(|| { - server_error!( - "chat response requested prompt_logprobs but generation returned none" - ) - })?, + Some(prompt_logprobs_to_maps( + prompt_logprobs.as_ref(), + &prompt_token_ids, return_tokens_as_token_ids, - )) + )?) } else { None }; diff --git a/rust/src/server/src/routes/openai/completions.rs b/rust/src/server/src/routes/openai/completions.rs index cf15e138bf..b6f2138f12 100644 --- a/rust/src/server/src/routes/openai/completions.rs +++ b/rust/src/server/src/routes/openai/completions.rs @@ -5,7 +5,6 @@ mod convert; mod types; mod validate; -use std::collections::HashMap; use std::convert::Infallible; use std::result::Result; use std::sync::Arc; @@ -29,8 +28,8 @@ use vllm_text::{ use self::convert::{ResponseOptions, prepare_completion_request}; use super::utils::logprobs::{ - collected_logprobs_to_openai, decoded_logprobs_to_openai, decoded_prompt_logprobs_to_maps, - decoded_prompt_logprobs_to_openai, text_len, + collected_logprobs_to_openai, decoded_logprobs_to_openai, decoded_prompt_logprobs_to_openai, + prompt_logprobs_to_maps, text_len, }; use super::utils::types::Usage; use crate::config::ApiServerOptions; @@ -505,27 +504,6 @@ fn prompt_only_logprobs_to_openai( )) } -fn prompt_logprobs_to_maps( - prompt_logprobs: Option<&DecodedPromptLogprobs>, - prompt_token_ids: &[u32], - return_tokens_as_token_ids: bool, -) -> Result>>, ApiError> { - if let Some(prompt_logprobs) = prompt_logprobs { - return Ok(decoded_prompt_logprobs_to_maps( - prompt_logprobs, - return_tokens_as_token_ids, - )); - } - - if let [_token_id] = prompt_token_ids { - return Ok(vec![None]); - } - - Err(server_error!( - "completion response requested prompt_logprobs but generation returned none" - )) -} - fn usage_chunk( request_id: &str, response_model: &str, diff --git a/rust/src/server/src/routes/openai/utils/logprobs.rs b/rust/src/server/src/routes/openai/utils/logprobs.rs index 078c42b507..e300e124b1 100644 --- a/rust/src/server/src/routes/openai/utils/logprobs.rs +++ b/rust/src/server/src/routes/openai/utils/logprobs.rs @@ -100,20 +100,31 @@ pub fn decoded_prompt_logprobs_to_openai( }) } -/// Convert decoded prompt logprobs into the vLLM-style prompt-logprobs response -/// shape. -pub fn decoded_prompt_logprobs_to_maps( - prompt_logprobs: &DecodedPromptLogprobs, +/// Map decoded prompt logprobs into vLLM-style per-position maps, treating a +/// missing single-token payload as `[None]`. +pub fn prompt_logprobs_to_maps( + prompt_logprobs: Option<&DecodedPromptLogprobs>, + prompt_token_ids: &[u32], return_tokens_as_token_ids: bool, -) -> Vec>> { - std::iter::once(None) - .chain(prompt_logprobs.scored_positions.iter().map(|position| { - Some(position_top_logprobs_map( - position, - return_tokens_as_token_ids, - )) - })) - .collect() +) -> Result>>, ApiError> { + if let Some(prompt_logprobs) = prompt_logprobs { + return Ok(std::iter::once(None) + .chain(prompt_logprobs.scored_positions.iter().map(|position| { + Some(position_top_logprobs_map( + position, + return_tokens_as_token_ids, + )) + })) + .collect()); + } + + if let [_token_id] = prompt_token_ids { + return Ok(vec![None]); + } + + Err(server_error!( + "prompt_logprobs were requested but generation returned none" + )) } /// Convert decoded token-position logprobs into the OpenAI chat `logprobs` @@ -275,7 +286,13 @@ pub fn clamp_logprob(logprob: f32) -> f32 { mod tests { use vllm_text::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTokenLogprob}; - use super::decoded_logprobs_to_openai_chat; + use super::{decoded_logprobs_to_openai_chat, prompt_logprobs_to_maps}; + + #[test] + fn prompt_logprobs_maps_reject_missing_multi_token_payload() { + prompt_logprobs_to_maps(None, &[9707, 11], false) + .expect_err("multi-token prompt without payload is an engine failure"); + } fn sample_logprobs() -> DecodedLogprobs { DecodedLogprobs {