[Bugfix][Rust Frontend] Map missing prompt logprobs for single-token prompts in chat and raw generate (#49111)
Signed-off-by: Feathbow <feathbow@gmail.com>
This commit is contained in:
@@ -238,13 +238,18 @@ fn collect_generate(
|
||||
None
|
||||
};
|
||||
let prompt_logprobs = if include_prompt_logprobs {
|
||||
let prompt_logprobs = collected.prompt_logprobs.as_ref().ok_or_else(|| {
|
||||
ApiError::server_error(
|
||||
"raw generate response requested prompt_logprobs but generation returned none"
|
||||
.to_string(),
|
||||
)
|
||||
})?;
|
||||
Some(raw_prompt_logprobs_to_maps(prompt_logprobs))
|
||||
match collected.prompt_logprobs.as_ref() {
|
||||
Some(prompt_logprobs) => Some(raw_prompt_logprobs_to_maps(prompt_logprobs)),
|
||||
// A single-token prompt has no scored positions; same mapping
|
||||
// as /v1/completions.
|
||||
None if collected.prompt_token_ids.len() == 1 => Some(vec![None]),
|
||||
None => {
|
||||
return Err(ApiError::server_error(
|
||||
"raw generate response requested prompt_logprobs but generation returned none"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -472,4 +477,48 @@ mod tests {
|
||||
Some(2)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collect_generate_maps_prompt_logprobs_for_single_token_prompt() {
|
||||
let output_without_payload = |prompt_token_ids: Vec<u32>| CollectedGenerateOutput {
|
||||
request_id: "raw-1".to_string(),
|
||||
prompt_logprobs: None,
|
||||
token_ids: vec![3],
|
||||
logprobs: None,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
usage: vllm_llm::TokenUsage {
|
||||
prompt_token_count: prompt_token_ids.len(),
|
||||
output_token_count: 1,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
prompt_token_ids,
|
||||
};
|
||||
|
||||
let response = collect_generate(
|
||||
output_without_payload(vec![9707]),
|
||||
"raw-1".to_string(),
|
||||
ApiServerOptions::default(),
|
||||
ResponseOptions {
|
||||
include_prompt_logprobs: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.expect("single-token prompt without payload maps to [None]");
|
||||
let prompt_logprobs = response.prompt_logprobs.expect("prompt logprobs present");
|
||||
assert_eq!(prompt_logprobs.len(), 1);
|
||||
assert!(prompt_logprobs[0].is_none());
|
||||
|
||||
collect_generate(
|
||||
output_without_payload(vec![9707, 11]),
|
||||
"raw-2".to_string(),
|
||||
ApiServerOptions::default(),
|
||||
ResponseOptions {
|
||||
include_prompt_logprobs: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.expect_err("multi-token prompt without payload is an engine failure");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,7 +35,7 @@ use crate::routes::openai::chat_completions::types::{
|
||||
ChatMessageDelta,
|
||||
};
|
||||
use crate::routes::openai::utils::logprobs::{
|
||||
decoded_logprobs_to_openai_chat, decoded_prompt_logprobs_to_maps,
|
||||
decoded_logprobs_to_openai_chat, prompt_logprobs_to_maps,
|
||||
};
|
||||
use crate::routes::openai::utils::types::{
|
||||
ChatLogProbs, FunctionCallDelta, FunctionCallResponse, ToolCall, ToolCallDelta, Usage,
|
||||
@@ -181,14 +181,11 @@ async fn collect_chat_completion(
|
||||
None
|
||||
};
|
||||
let prompt_logprobs = if include_prompt_logprobs {
|
||||
Some(decoded_prompt_logprobs_to_maps(
|
||||
prompt_logprobs.as_ref().ok_or_else(|| {
|
||||
server_error!(
|
||||
"chat response requested prompt_logprobs but generation returned none"
|
||||
)
|
||||
})?,
|
||||
Some(prompt_logprobs_to_maps(
|
||||
prompt_logprobs.as_ref(),
|
||||
&prompt_token_ids,
|
||||
return_tokens_as_token_ids,
|
||||
))
|
||||
)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
@@ -5,7 +5,6 @@ mod convert;
|
||||
mod types;
|
||||
mod validate;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::convert::Infallible;
|
||||
use std::result::Result;
|
||||
use std::sync::Arc;
|
||||
@@ -29,8 +28,8 @@ use vllm_text::{
|
||||
|
||||
use self::convert::{ResponseOptions, prepare_completion_request};
|
||||
use super::utils::logprobs::{
|
||||
collected_logprobs_to_openai, decoded_logprobs_to_openai, decoded_prompt_logprobs_to_maps,
|
||||
decoded_prompt_logprobs_to_openai, text_len,
|
||||
collected_logprobs_to_openai, decoded_logprobs_to_openai, decoded_prompt_logprobs_to_openai,
|
||||
prompt_logprobs_to_maps, text_len,
|
||||
};
|
||||
use super::utils::types::Usage;
|
||||
use crate::config::ApiServerOptions;
|
||||
@@ -505,27 +504,6 @@ fn prompt_only_logprobs_to_openai(
|
||||
))
|
||||
}
|
||||
|
||||
fn prompt_logprobs_to_maps(
|
||||
prompt_logprobs: Option<&DecodedPromptLogprobs>,
|
||||
prompt_token_ids: &[u32],
|
||||
return_tokens_as_token_ids: bool,
|
||||
) -> Result<Vec<Option<HashMap<String, f32>>>, ApiError> {
|
||||
if let Some(prompt_logprobs) = prompt_logprobs {
|
||||
return Ok(decoded_prompt_logprobs_to_maps(
|
||||
prompt_logprobs,
|
||||
return_tokens_as_token_ids,
|
||||
));
|
||||
}
|
||||
|
||||
if let [_token_id] = prompt_token_ids {
|
||||
return Ok(vec![None]);
|
||||
}
|
||||
|
||||
Err(server_error!(
|
||||
"completion response requested prompt_logprobs but generation returned none"
|
||||
))
|
||||
}
|
||||
|
||||
fn usage_chunk(
|
||||
request_id: &str,
|
||||
response_model: &str,
|
||||
|
||||
@@ -100,20 +100,31 @@ pub fn decoded_prompt_logprobs_to_openai(
|
||||
})
|
||||
}
|
||||
|
||||
/// Convert decoded prompt logprobs into the vLLM-style prompt-logprobs response
|
||||
/// shape.
|
||||
pub fn decoded_prompt_logprobs_to_maps(
|
||||
prompt_logprobs: &DecodedPromptLogprobs,
|
||||
/// Map decoded prompt logprobs into vLLM-style per-position maps, treating a
|
||||
/// missing single-token payload as `[None]`.
|
||||
pub fn prompt_logprobs_to_maps(
|
||||
prompt_logprobs: Option<&DecodedPromptLogprobs>,
|
||||
prompt_token_ids: &[u32],
|
||||
return_tokens_as_token_ids: bool,
|
||||
) -> Vec<Option<HashMap<String, f32>>> {
|
||||
std::iter::once(None)
|
||||
.chain(prompt_logprobs.scored_positions.iter().map(|position| {
|
||||
Some(position_top_logprobs_map(
|
||||
position,
|
||||
return_tokens_as_token_ids,
|
||||
))
|
||||
}))
|
||||
.collect()
|
||||
) -> Result<Vec<Option<HashMap<String, f32>>>, ApiError> {
|
||||
if let Some(prompt_logprobs) = prompt_logprobs {
|
||||
return Ok(std::iter::once(None)
|
||||
.chain(prompt_logprobs.scored_positions.iter().map(|position| {
|
||||
Some(position_top_logprobs_map(
|
||||
position,
|
||||
return_tokens_as_token_ids,
|
||||
))
|
||||
}))
|
||||
.collect());
|
||||
}
|
||||
|
||||
if let [_token_id] = prompt_token_ids {
|
||||
return Ok(vec![None]);
|
||||
}
|
||||
|
||||
Err(server_error!(
|
||||
"prompt_logprobs were requested but generation returned none"
|
||||
))
|
||||
}
|
||||
|
||||
/// Convert decoded token-position logprobs into the OpenAI chat `logprobs`
|
||||
@@ -275,7 +286,13 @@ pub fn clamp_logprob(logprob: f32) -> f32 {
|
||||
mod tests {
|
||||
use vllm_text::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTokenLogprob};
|
||||
|
||||
use super::decoded_logprobs_to_openai_chat;
|
||||
use super::{decoded_logprobs_to_openai_chat, prompt_logprobs_to_maps};
|
||||
|
||||
#[test]
|
||||
fn prompt_logprobs_maps_reject_missing_multi_token_payload() {
|
||||
prompt_logprobs_to_maps(None, &[9707, 11], false)
|
||||
.expect_err("multi-token prompt without payload is an engine failure");
|
||||
}
|
||||
|
||||
fn sample_logprobs() -> DecodedLogprobs {
|
||||
DecodedLogprobs {
|
||||
|
||||
Reference in New Issue
Block a user