[Frontend] Return prompt metadata from /inference/v1/generate (#53187)

This commit is contained in:
aoshen02
2026-09-16 15:29:07 -07:00
committed by GitHub
parent 6c3d9816af
commit 0983aef8da
9 changed files with 763 additions and 40 deletions
@@ -21,6 +21,7 @@ use thiserror_ext::AsReport as _;
use tracing::{error, info, trace};
use tracing_futures::Instrument as _;
use vllm_engine_core_client::protocol::logprobs::{Logprobs, PositionLogprobs};
use vllm_engine_core_client::protocol::multimodal::MmFeatureSpec;
use vllm_llm::{
CollectedGenerateOutput, FinishReason, GenerateOutput, GenerateOutputStreamExt as _, TokenUsage,
};
@@ -28,7 +29,7 @@ use vllm_llm::{
use self::convert::{ResponseOptions, prepare_generate_request};
use self::types::{
GenerateLogprob, GenerateResponse, GenerateResponseChoice, GenerateResponseStreamChoice,
GenerateStreamResponse,
GenerateStreamResponse, MultiModalPlaceholders, PlaceholderRangeInfo,
};
pub(crate) use self::types::{GenerateRequest, GenerateSamplingParams};
pub(crate) use self::validate::validate_request_compat;
@@ -64,6 +65,11 @@ pub async fn generate(
} else {
None
};
let mm_placeholders = body
.return_token_ids
.unwrap_or(false)
.then(|| extract_mm_placeholders(mm_features.as_deref()))
.flatten();
let prepared =
match prepare_generate_request(body, &lora_resolution, request_context, mm_features) {
@@ -98,6 +104,7 @@ pub async fn generate(
prepared.request_id,
api_server_options,
prepared.options,
mm_placeholders,
);
let sse_stream = generate_sse_stream(chunk_stream).instrument(request_span);
@@ -120,6 +127,7 @@ pub async fn generate(
prepared.request_id,
api_server_options,
prepared.options,
mm_placeholders,
) {
Ok(response) => response,
Err(error) => return error.into_response(),
@@ -143,19 +151,26 @@ async fn generate_chunk_stream(
include_logprobs,
// Ignored: raw generate streaming has no prompt-logprobs wire shape.
include_prompt_logprobs: _,
return_token_ids,
}: ResponseOptions,
mut mm_placeholders: Option<MultiModalPlaceholders>,
mut y: TryYielder<GenerateStreamResponse, ApiError>,
) -> Result<(), ApiError> {
pin_mut!(stream);
let mut prompt_tokens = None;
let mut prompt_token_ids = None;
let mut usage = TokenUsage::default();
while let Some(next) = stream.next().await {
match next {
Ok(output) => {
if prompt_tokens.is_none() {
prompt_tokens =
output.prompt_info.as_ref().map(|info| info.prompt_token_ids.len());
if prompt_tokens.is_none()
&& let Some(info) = output.prompt_info.as_ref()
{
prompt_tokens = Some(info.prompt_token_ids.len());
if return_token_ids {
prompt_token_ids = Some(info.prompt_token_ids.to_vec());
}
}
usage.prompt_token_count = prompt_tokens.unwrap_or_default();
usage.cached_token_count = usage.cached_token_count.max(output.cached_token_count);
@@ -195,6 +210,7 @@ async fn generate_chunk_stream(
None
};
let prompt_token_ids = prompt_token_ids.take();
y.yield_ok(GenerateStreamResponse {
request_id: request_id.clone(),
choices: vec![GenerateResponseStreamChoice {
@@ -205,6 +221,8 @@ async fn generate_chunk_stream(
}],
usage: include_continuous_usage
.then(|| Usage::from_token_usage(usage, enable_prompt_tokens_details)),
mm_placeholders: prompt_token_ids.as_ref().and_then(|_| mm_placeholders.take()),
prompt_token_ids,
})
.await;
}
@@ -223,6 +241,8 @@ async fn generate_chunk_stream(
request_id,
choices: Vec::new(),
usage: Some(Usage::from_token_usage(usage, enable_prompt_tokens_details)),
prompt_token_ids: None,
mm_placeholders: None,
})
.await;
}
@@ -244,7 +264,9 @@ fn collect_generate(
include_continuous_usage: _,
include_logprobs,
include_prompt_logprobs,
return_token_ids,
}: ResponseOptions,
mm_placeholders: Option<MultiModalPlaceholders>,
) -> Result<GenerateResponse, ApiError> {
let logprobs = if include_logprobs {
let logprobs = collected.logprobs.as_ref().ok_or_else(|| {
@@ -292,11 +314,34 @@ fn collect_generate(
token_ids: collected.token_ids,
}],
prompt_logprobs,
prompt_token_ids: return_token_ids.then_some(collected.prompt_token_ids),
mm_placeholders: return_token_ids.then_some(mm_placeholders).flatten(),
kv_transfer_params: collected.kv_transfer_params,
ec_transfer_params: collected.ec_transfer_params,
})
}
fn extract_mm_placeholders(features: Option<&[MmFeatureSpec]>) -> Option<MultiModalPlaceholders> {
let features = features?;
if features.is_empty() {
return None;
}
let mut placeholders = MultiModalPlaceholders::new();
for feature in features {
placeholders.entry(feature.modality.as_str().to_string()).or_default().push(
PlaceholderRangeInfo {
offset: feature.mm_position.offset,
length: feature.mm_position.length,
},
);
}
for ranges in placeholders.values_mut() {
ranges.sort_by_key(|range| range.offset);
}
Some(placeholders)
}
fn raw_logprobs_to_openai_chat(logprobs: &Logprobs) -> Result<ChatLogProbs, ApiError> {
let content = logprobs
.positions
@@ -419,6 +464,7 @@ mod tests {
use std::sync::Arc;
use futures::{TryStreamExt as _, stream};
use vllm_engine_core_client::protocol::multimodal::{MmModality, PlaceholderRange};
use vllm_llm::GeneratePromptInfo;
use super::*;
@@ -451,6 +497,13 @@ mod tests {
}),
]);
let mm_placeholders = MultiModalPlaceholders::from([(
"image".to_string(),
vec![PlaceholderRangeInfo {
offset: 1,
length: 4,
}],
)]);
let chunks: Vec<_> = generate_chunk_stream(
stream,
"raw-stream".to_string(),
@@ -461,14 +514,20 @@ mod tests {
ResponseOptions {
include_usage: true,
include_continuous_usage: true,
return_token_ids: true,
..Default::default()
},
Some(mm_placeholders.clone()),
)
.try_collect()
.await
.expect("collect chunks");
assert_eq!(chunks.len(), 2);
assert_eq!(chunks[0].prompt_token_ids, Some(vec![11, 22]));
assert_eq!(chunks[0].mm_placeholders, Some(mm_placeholders));
assert!(chunks[1].prompt_token_ids.is_none());
assert!(chunks[1].mm_placeholders.is_none());
assert_eq!(
chunks[0].usage.as_ref().expect("chunk usage").prompt_tokens,
2
@@ -499,6 +558,209 @@ mod tests {
);
}
fn stream_output(
prompt_token_ids: Option<&[u32]>,
token_ids: Vec<u32>,
finish_reason: Option<FinishReason>,
) -> GenerateOutput {
GenerateOutput {
request_id: String::new(),
prompt_info: prompt_token_ids.map(|ids| GeneratePromptInfo {
prompt_token_ids: Arc::from(ids),
prompt_logprobs: None,
}),
token_ids,
logprobs: None,
finish_reason,
cached_token_count: 0,
kv_transfer_params: None,
ec_transfer_params: None,
}
}
async fn collect_chunks(
outputs: Vec<GenerateOutput>,
return_token_ids: bool,
mm_placeholders: Option<MultiModalPlaceholders>,
) -> Vec<GenerateStreamResponse> {
generate_chunk_stream(
stream::iter(outputs.into_iter().map(Ok)),
"raw-stream".to_string(),
ApiServerOptions::default(),
ResponseOptions {
return_token_ids,
..Default::default()
},
mm_placeholders,
)
.try_collect()
.await
.expect("collect chunks")
}
#[tokio::test]
async fn generate_chunk_stream_omits_prompt_metadata_by_default() {
let chunks = collect_chunks(
vec![
stream_output(Some(&[11, 22]), vec![33], None),
stream_output(Some(&[11, 22]), vec![44], Some(FinishReason::stop_eos())),
],
false,
Some(MultiModalPlaceholders::from([(
"image".to_string(),
vec![PlaceholderRangeInfo {
offset: 1,
length: 4,
}],
)])),
)
.await;
assert_eq!(chunks.len(), 2);
assert!(chunks.iter().all(|c| c.prompt_token_ids.is_none()));
assert!(chunks.iter().all(|c| c.mm_placeholders.is_none()));
}
#[tokio::test]
async fn generate_chunk_stream_sends_prompt_metadata_on_first_emitted_chunk() {
// An empty first output is skipped and must not consume the metadata.
let chunks = collect_chunks(
vec![
stream_output(Some(&[11, 22]), Vec::new(), None),
stream_output(Some(&[11, 22]), vec![33], None),
stream_output(Some(&[11, 22]), vec![44], Some(FinishReason::stop_eos())),
],
true,
None,
)
.await;
assert_eq!(chunks.len(), 2);
assert_eq!(chunks[0].prompt_token_ids, Some(vec![11, 22]));
assert!(chunks[0].mm_placeholders.is_none());
assert!(chunks[1].prompt_token_ids.is_none());
assert!(chunks[1].mm_placeholders.is_none());
}
#[test]
fn collect_generate_omits_prompt_metadata_by_default() {
let output = CollectedGenerateOutput {
request_id: "raw-1".to_string(),
prompt_logprobs: None,
token_ids: vec![30],
logprobs: None,
finish_reason: FinishReason::stop_eos(),
usage: TokenUsage {
prompt_token_count: 2,
output_token_count: 1,
cached_token_count: 0,
},
kv_transfer_params: None,
ec_transfer_params: None,
prompt_token_ids: vec![10, 20],
};
let response = collect_generate(
output,
"raw-1".to_string(),
ApiServerOptions::default(),
ResponseOptions::default(),
Some(MultiModalPlaceholders::from([(
"image".to_string(),
vec![PlaceholderRangeInfo {
offset: 0,
length: 1,
}],
)])),
)
.expect("response");
assert!(response.prompt_token_ids.is_none());
assert!(response.mm_placeholders.is_none());
}
#[test]
fn extract_mm_placeholders_returns_none_without_features() {
assert!(extract_mm_placeholders(None).is_none());
assert!(extract_mm_placeholders(Some(&[])).is_none());
}
#[test]
fn extract_mm_placeholders_groups_ranges_in_prompt_order() {
let feature = |modality: MmModality, offset: usize, length: usize| MmFeatureSpec {
data: None,
modality,
identifier: format!("{}-{offset}", modality.as_str()),
mm_position: PlaceholderRange {
offset,
length,
is_embed: None,
},
mm_hash: None,
};
let features = vec![
feature(MmModality::Image, 8, 2),
feature(MmModality::Audio, 4, 3),
feature(MmModality::Image, 1, 5),
];
let placeholders = extract_mm_placeholders(Some(&features)).expect("metadata");
assert_eq!(
placeholders["image"],
vec![
PlaceholderRangeInfo {
offset: 1,
length: 5,
},
PlaceholderRangeInfo {
offset: 8,
length: 2,
},
]
);
assert_eq!(
placeholders["audio"],
vec![PlaceholderRangeInfo {
offset: 4,
length: 3,
}]
);
}
#[test]
fn collect_generate_returns_prompt_token_ids_when_requested() {
let output = CollectedGenerateOutput {
request_id: "raw-1".to_string(),
prompt_logprobs: None,
token_ids: vec![30],
logprobs: None,
finish_reason: FinishReason::stop_eos(),
usage: TokenUsage {
prompt_token_count: 2,
output_token_count: 1,
cached_token_count: 0,
},
kv_transfer_params: None,
ec_transfer_params: None,
prompt_token_ids: vec![10, 20],
};
let response = collect_generate(
output,
"raw-1".to_string(),
ApiServerOptions::default(),
ResponseOptions {
return_token_ids: true,
..Default::default()
},
None,
)
.expect("response");
assert_eq!(response.prompt_token_ids, Some(vec![10, 20]));
}
#[test]
fn collect_generate_maps_prompt_logprobs_for_single_token_prompt() {
let output_without_payload = |prompt_token_ids: Vec<u32>| CollectedGenerateOutput {
@@ -525,6 +787,7 @@ mod tests {
include_prompt_logprobs: true,
..Default::default()
},
None,
)
.expect("single-token prompt without payload maps to [None]");
let prompt_logprobs = response.prompt_logprobs.expect("prompt logprobs present");
@@ -539,6 +802,7 @@ mod tests {
include_prompt_logprobs: true,
..Default::default()
},
None,
)
.expect_err("multi-token prompt without payload is an engine failure");
}
@@ -30,6 +30,8 @@ pub(super) struct ResponseOptions {
pub include_logprobs: bool,
/// Whether the caller requested top-level prompt logprobs.
pub include_prompt_logprobs: bool,
/// Whether the caller requested final prompt token metadata.
pub return_token_ids: bool,
}
/// Validate and lower one raw generate request into the internal
@@ -56,6 +58,7 @@ pub(super) fn prepare_generate_request(
.unwrap_or(false);
let include_logprobs = request.sampling_params.inner.logprobs.is_some();
let include_prompt_logprobs = request.sampling_params.inner.prompt_logprobs.is_some();
let return_token_ids = request.return_token_ids.unwrap_or(false);
let mut sampling_params = request.sampling_params.inner;
sampling_params.vllm_xargs = merge_kv_transfer_params(
sampling_params.vllm_xargs,
@@ -93,6 +96,7 @@ pub(super) fn prepare_generate_request(
include_continuous_usage,
include_logprobs,
include_prompt_logprobs,
return_token_ids,
},
})
}
@@ -46,6 +46,7 @@ pub struct GenerateRequest {
pub ec_transfer_params: Option<HashMap<String, Value>>,
/// Raw multimodal input; server resolves media. Mutually exclusive with `features`.
pub content_parts: Option<Vec<MediaContentPart>>,
pub return_token_ids: Option<bool>,
#[serde(flatten)]
pub other: Map<String, Value>,
}
@@ -81,6 +82,8 @@ pub(super) struct GenerateStreamResponse {
pub request_id: String,
pub choices: Vec<GenerateResponseStreamChoice>,
pub usage: Option<Usage>,
pub prompt_token_ids: Option<Vec<u32>>,
pub mm_placeholders: Option<MultiModalPlaceholders>,
}
/// Mirrors the Python vLLM `GenerateResponse` class.
@@ -89,10 +92,20 @@ pub(super) struct GenerateResponse {
pub request_id: String,
pub choices: Vec<GenerateResponseChoice>,
pub prompt_logprobs: Option<Vec<Option<HashMap<u32, GenerateLogprob>>>>,
pub prompt_token_ids: Option<Vec<u32>>,
pub mm_placeholders: Option<MultiModalPlaceholders>,
pub kv_transfer_params: Option<Value>,
pub ec_transfer_params: Option<Value>,
}
pub(super) type MultiModalPlaceholders = HashMap<String, Vec<PlaceholderRangeInfo>>;
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub(super) struct PlaceholderRangeInfo {
pub offset: usize,
pub length: usize,
}
/// Mirrors the Python vLLM `Logprob` class used in prompt-logprobs payloads.
#[derive(Debug, Clone, Serialize)]
pub(super) struct GenerateLogprob {
+1
View File
@@ -105,6 +105,7 @@ fn lower_render_request(
kv_transfer_params: None,
ec_transfer_params: None,
content_parts: None,
return_token_ids: None,
other: Default::default(),
};
validate_generate_request(&request, &state.served_model_names)?;