[Frontend] Return prompt metadata from /inference/v1/generate (#53187)
This commit is contained in:
@@ -21,6 +21,7 @@ use thiserror_ext::AsReport as _;
|
||||
use tracing::{error, info, trace};
|
||||
use tracing_futures::Instrument as _;
|
||||
use vllm_engine_core_client::protocol::logprobs::{Logprobs, PositionLogprobs};
|
||||
use vllm_engine_core_client::protocol::multimodal::MmFeatureSpec;
|
||||
use vllm_llm::{
|
||||
CollectedGenerateOutput, FinishReason, GenerateOutput, GenerateOutputStreamExt as _, TokenUsage,
|
||||
};
|
||||
@@ -28,7 +29,7 @@ use vllm_llm::{
|
||||
use self::convert::{ResponseOptions, prepare_generate_request};
|
||||
use self::types::{
|
||||
GenerateLogprob, GenerateResponse, GenerateResponseChoice, GenerateResponseStreamChoice,
|
||||
GenerateStreamResponse,
|
||||
GenerateStreamResponse, MultiModalPlaceholders, PlaceholderRangeInfo,
|
||||
};
|
||||
pub(crate) use self::types::{GenerateRequest, GenerateSamplingParams};
|
||||
pub(crate) use self::validate::validate_request_compat;
|
||||
@@ -64,6 +65,11 @@ pub async fn generate(
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mm_placeholders = body
|
||||
.return_token_ids
|
||||
.unwrap_or(false)
|
||||
.then(|| extract_mm_placeholders(mm_features.as_deref()))
|
||||
.flatten();
|
||||
|
||||
let prepared =
|
||||
match prepare_generate_request(body, &lora_resolution, request_context, mm_features) {
|
||||
@@ -98,6 +104,7 @@ pub async fn generate(
|
||||
prepared.request_id,
|
||||
api_server_options,
|
||||
prepared.options,
|
||||
mm_placeholders,
|
||||
);
|
||||
let sse_stream = generate_sse_stream(chunk_stream).instrument(request_span);
|
||||
|
||||
@@ -120,6 +127,7 @@ pub async fn generate(
|
||||
prepared.request_id,
|
||||
api_server_options,
|
||||
prepared.options,
|
||||
mm_placeholders,
|
||||
) {
|
||||
Ok(response) => response,
|
||||
Err(error) => return error.into_response(),
|
||||
@@ -143,19 +151,26 @@ async fn generate_chunk_stream(
|
||||
include_logprobs,
|
||||
// Ignored: raw generate streaming has no prompt-logprobs wire shape.
|
||||
include_prompt_logprobs: _,
|
||||
return_token_ids,
|
||||
}: ResponseOptions,
|
||||
mut mm_placeholders: Option<MultiModalPlaceholders>,
|
||||
mut y: TryYielder<GenerateStreamResponse, ApiError>,
|
||||
) -> Result<(), ApiError> {
|
||||
pin_mut!(stream);
|
||||
let mut prompt_tokens = None;
|
||||
let mut prompt_token_ids = None;
|
||||
let mut usage = TokenUsage::default();
|
||||
|
||||
while let Some(next) = stream.next().await {
|
||||
match next {
|
||||
Ok(output) => {
|
||||
if prompt_tokens.is_none() {
|
||||
prompt_tokens =
|
||||
output.prompt_info.as_ref().map(|info| info.prompt_token_ids.len());
|
||||
if prompt_tokens.is_none()
|
||||
&& let Some(info) = output.prompt_info.as_ref()
|
||||
{
|
||||
prompt_tokens = Some(info.prompt_token_ids.len());
|
||||
if return_token_ids {
|
||||
prompt_token_ids = Some(info.prompt_token_ids.to_vec());
|
||||
}
|
||||
}
|
||||
usage.prompt_token_count = prompt_tokens.unwrap_or_default();
|
||||
usage.cached_token_count = usage.cached_token_count.max(output.cached_token_count);
|
||||
@@ -195,6 +210,7 @@ async fn generate_chunk_stream(
|
||||
None
|
||||
};
|
||||
|
||||
let prompt_token_ids = prompt_token_ids.take();
|
||||
y.yield_ok(GenerateStreamResponse {
|
||||
request_id: request_id.clone(),
|
||||
choices: vec![GenerateResponseStreamChoice {
|
||||
@@ -205,6 +221,8 @@ async fn generate_chunk_stream(
|
||||
}],
|
||||
usage: include_continuous_usage
|
||||
.then(|| Usage::from_token_usage(usage, enable_prompt_tokens_details)),
|
||||
mm_placeholders: prompt_token_ids.as_ref().and_then(|_| mm_placeholders.take()),
|
||||
prompt_token_ids,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
@@ -223,6 +241,8 @@ async fn generate_chunk_stream(
|
||||
request_id,
|
||||
choices: Vec::new(),
|
||||
usage: Some(Usage::from_token_usage(usage, enable_prompt_tokens_details)),
|
||||
prompt_token_ids: None,
|
||||
mm_placeholders: None,
|
||||
})
|
||||
.await;
|
||||
}
|
||||
@@ -244,7 +264,9 @@ fn collect_generate(
|
||||
include_continuous_usage: _,
|
||||
include_logprobs,
|
||||
include_prompt_logprobs,
|
||||
return_token_ids,
|
||||
}: ResponseOptions,
|
||||
mm_placeholders: Option<MultiModalPlaceholders>,
|
||||
) -> Result<GenerateResponse, ApiError> {
|
||||
let logprobs = if include_logprobs {
|
||||
let logprobs = collected.logprobs.as_ref().ok_or_else(|| {
|
||||
@@ -292,11 +314,34 @@ fn collect_generate(
|
||||
token_ids: collected.token_ids,
|
||||
}],
|
||||
prompt_logprobs,
|
||||
prompt_token_ids: return_token_ids.then_some(collected.prompt_token_ids),
|
||||
mm_placeholders: return_token_ids.then_some(mm_placeholders).flatten(),
|
||||
kv_transfer_params: collected.kv_transfer_params,
|
||||
ec_transfer_params: collected.ec_transfer_params,
|
||||
})
|
||||
}
|
||||
|
||||
fn extract_mm_placeholders(features: Option<&[MmFeatureSpec]>) -> Option<MultiModalPlaceholders> {
|
||||
let features = features?;
|
||||
if features.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut placeholders = MultiModalPlaceholders::new();
|
||||
for feature in features {
|
||||
placeholders.entry(feature.modality.as_str().to_string()).or_default().push(
|
||||
PlaceholderRangeInfo {
|
||||
offset: feature.mm_position.offset,
|
||||
length: feature.mm_position.length,
|
||||
},
|
||||
);
|
||||
}
|
||||
for ranges in placeholders.values_mut() {
|
||||
ranges.sort_by_key(|range| range.offset);
|
||||
}
|
||||
Some(placeholders)
|
||||
}
|
||||
|
||||
fn raw_logprobs_to_openai_chat(logprobs: &Logprobs) -> Result<ChatLogProbs, ApiError> {
|
||||
let content = logprobs
|
||||
.positions
|
||||
@@ -419,6 +464,7 @@ mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use futures::{TryStreamExt as _, stream};
|
||||
use vllm_engine_core_client::protocol::multimodal::{MmModality, PlaceholderRange};
|
||||
use vllm_llm::GeneratePromptInfo;
|
||||
|
||||
use super::*;
|
||||
@@ -451,6 +497,13 @@ mod tests {
|
||||
}),
|
||||
]);
|
||||
|
||||
let mm_placeholders = MultiModalPlaceholders::from([(
|
||||
"image".to_string(),
|
||||
vec![PlaceholderRangeInfo {
|
||||
offset: 1,
|
||||
length: 4,
|
||||
}],
|
||||
)]);
|
||||
let chunks: Vec<_> = generate_chunk_stream(
|
||||
stream,
|
||||
"raw-stream".to_string(),
|
||||
@@ -461,14 +514,20 @@ mod tests {
|
||||
ResponseOptions {
|
||||
include_usage: true,
|
||||
include_continuous_usage: true,
|
||||
return_token_ids: true,
|
||||
..Default::default()
|
||||
},
|
||||
Some(mm_placeholders.clone()),
|
||||
)
|
||||
.try_collect()
|
||||
.await
|
||||
.expect("collect chunks");
|
||||
|
||||
assert_eq!(chunks.len(), 2);
|
||||
assert_eq!(chunks[0].prompt_token_ids, Some(vec![11, 22]));
|
||||
assert_eq!(chunks[0].mm_placeholders, Some(mm_placeholders));
|
||||
assert!(chunks[1].prompt_token_ids.is_none());
|
||||
assert!(chunks[1].mm_placeholders.is_none());
|
||||
assert_eq!(
|
||||
chunks[0].usage.as_ref().expect("chunk usage").prompt_tokens,
|
||||
2
|
||||
@@ -499,6 +558,209 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
fn stream_output(
|
||||
prompt_token_ids: Option<&[u32]>,
|
||||
token_ids: Vec<u32>,
|
||||
finish_reason: Option<FinishReason>,
|
||||
) -> GenerateOutput {
|
||||
GenerateOutput {
|
||||
request_id: String::new(),
|
||||
prompt_info: prompt_token_ids.map(|ids| GeneratePromptInfo {
|
||||
prompt_token_ids: Arc::from(ids),
|
||||
prompt_logprobs: None,
|
||||
}),
|
||||
token_ids,
|
||||
logprobs: None,
|
||||
finish_reason,
|
||||
cached_token_count: 0,
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
}
|
||||
}
|
||||
|
||||
async fn collect_chunks(
|
||||
outputs: Vec<GenerateOutput>,
|
||||
return_token_ids: bool,
|
||||
mm_placeholders: Option<MultiModalPlaceholders>,
|
||||
) -> Vec<GenerateStreamResponse> {
|
||||
generate_chunk_stream(
|
||||
stream::iter(outputs.into_iter().map(Ok)),
|
||||
"raw-stream".to_string(),
|
||||
ApiServerOptions::default(),
|
||||
ResponseOptions {
|
||||
return_token_ids,
|
||||
..Default::default()
|
||||
},
|
||||
mm_placeholders,
|
||||
)
|
||||
.try_collect()
|
||||
.await
|
||||
.expect("collect chunks")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn generate_chunk_stream_omits_prompt_metadata_by_default() {
|
||||
let chunks = collect_chunks(
|
||||
vec![
|
||||
stream_output(Some(&[11, 22]), vec![33], None),
|
||||
stream_output(Some(&[11, 22]), vec![44], Some(FinishReason::stop_eos())),
|
||||
],
|
||||
false,
|
||||
Some(MultiModalPlaceholders::from([(
|
||||
"image".to_string(),
|
||||
vec![PlaceholderRangeInfo {
|
||||
offset: 1,
|
||||
length: 4,
|
||||
}],
|
||||
)])),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(chunks.len(), 2);
|
||||
assert!(chunks.iter().all(|c| c.prompt_token_ids.is_none()));
|
||||
assert!(chunks.iter().all(|c| c.mm_placeholders.is_none()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn generate_chunk_stream_sends_prompt_metadata_on_first_emitted_chunk() {
|
||||
// An empty first output is skipped and must not consume the metadata.
|
||||
let chunks = collect_chunks(
|
||||
vec![
|
||||
stream_output(Some(&[11, 22]), Vec::new(), None),
|
||||
stream_output(Some(&[11, 22]), vec![33], None),
|
||||
stream_output(Some(&[11, 22]), vec![44], Some(FinishReason::stop_eos())),
|
||||
],
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(chunks.len(), 2);
|
||||
assert_eq!(chunks[0].prompt_token_ids, Some(vec![11, 22]));
|
||||
assert!(chunks[0].mm_placeholders.is_none());
|
||||
assert!(chunks[1].prompt_token_ids.is_none());
|
||||
assert!(chunks[1].mm_placeholders.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collect_generate_omits_prompt_metadata_by_default() {
|
||||
let output = CollectedGenerateOutput {
|
||||
request_id: "raw-1".to_string(),
|
||||
prompt_logprobs: None,
|
||||
token_ids: vec![30],
|
||||
logprobs: None,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
usage: TokenUsage {
|
||||
prompt_token_count: 2,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
prompt_token_ids: vec![10, 20],
|
||||
};
|
||||
|
||||
let response = collect_generate(
|
||||
output,
|
||||
"raw-1".to_string(),
|
||||
ApiServerOptions::default(),
|
||||
ResponseOptions::default(),
|
||||
Some(MultiModalPlaceholders::from([(
|
||||
"image".to_string(),
|
||||
vec![PlaceholderRangeInfo {
|
||||
offset: 0,
|
||||
length: 1,
|
||||
}],
|
||||
)])),
|
||||
)
|
||||
.expect("response");
|
||||
|
||||
assert!(response.prompt_token_ids.is_none());
|
||||
assert!(response.mm_placeholders.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_mm_placeholders_returns_none_without_features() {
|
||||
assert!(extract_mm_placeholders(None).is_none());
|
||||
assert!(extract_mm_placeholders(Some(&[])).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_mm_placeholders_groups_ranges_in_prompt_order() {
|
||||
let feature = |modality: MmModality, offset: usize, length: usize| MmFeatureSpec {
|
||||
data: None,
|
||||
modality,
|
||||
identifier: format!("{}-{offset}", modality.as_str()),
|
||||
mm_position: PlaceholderRange {
|
||||
offset,
|
||||
length,
|
||||
is_embed: None,
|
||||
},
|
||||
mm_hash: None,
|
||||
};
|
||||
let features = vec![
|
||||
feature(MmModality::Image, 8, 2),
|
||||
feature(MmModality::Audio, 4, 3),
|
||||
feature(MmModality::Image, 1, 5),
|
||||
];
|
||||
|
||||
let placeholders = extract_mm_placeholders(Some(&features)).expect("metadata");
|
||||
|
||||
assert_eq!(
|
||||
placeholders["image"],
|
||||
vec![
|
||||
PlaceholderRangeInfo {
|
||||
offset: 1,
|
||||
length: 5,
|
||||
},
|
||||
PlaceholderRangeInfo {
|
||||
offset: 8,
|
||||
length: 2,
|
||||
},
|
||||
]
|
||||
);
|
||||
assert_eq!(
|
||||
placeholders["audio"],
|
||||
vec![PlaceholderRangeInfo {
|
||||
offset: 4,
|
||||
length: 3,
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collect_generate_returns_prompt_token_ids_when_requested() {
|
||||
let output = CollectedGenerateOutput {
|
||||
request_id: "raw-1".to_string(),
|
||||
prompt_logprobs: None,
|
||||
token_ids: vec![30],
|
||||
logprobs: None,
|
||||
finish_reason: FinishReason::stop_eos(),
|
||||
usage: TokenUsage {
|
||||
prompt_token_count: 2,
|
||||
output_token_count: 1,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
prompt_token_ids: vec![10, 20],
|
||||
};
|
||||
|
||||
let response = collect_generate(
|
||||
output,
|
||||
"raw-1".to_string(),
|
||||
ApiServerOptions::default(),
|
||||
ResponseOptions {
|
||||
return_token_ids: true,
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
)
|
||||
.expect("response");
|
||||
|
||||
assert_eq!(response.prompt_token_ids, Some(vec![10, 20]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collect_generate_maps_prompt_logprobs_for_single_token_prompt() {
|
||||
let output_without_payload = |prompt_token_ids: Vec<u32>| CollectedGenerateOutput {
|
||||
@@ -525,6 +787,7 @@ mod tests {
|
||||
include_prompt_logprobs: true,
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
)
|
||||
.expect("single-token prompt without payload maps to [None]");
|
||||
let prompt_logprobs = response.prompt_logprobs.expect("prompt logprobs present");
|
||||
@@ -539,6 +802,7 @@ mod tests {
|
||||
include_prompt_logprobs: true,
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
)
|
||||
.expect_err("multi-token prompt without payload is an engine failure");
|
||||
}
|
||||
|
||||
@@ -30,6 +30,8 @@ pub(super) struct ResponseOptions {
|
||||
pub include_logprobs: bool,
|
||||
/// Whether the caller requested top-level prompt logprobs.
|
||||
pub include_prompt_logprobs: bool,
|
||||
/// Whether the caller requested final prompt token metadata.
|
||||
pub return_token_ids: bool,
|
||||
}
|
||||
|
||||
/// Validate and lower one raw generate request into the internal
|
||||
@@ -56,6 +58,7 @@ pub(super) fn prepare_generate_request(
|
||||
.unwrap_or(false);
|
||||
let include_logprobs = request.sampling_params.inner.logprobs.is_some();
|
||||
let include_prompt_logprobs = request.sampling_params.inner.prompt_logprobs.is_some();
|
||||
let return_token_ids = request.return_token_ids.unwrap_or(false);
|
||||
let mut sampling_params = request.sampling_params.inner;
|
||||
sampling_params.vllm_xargs = merge_kv_transfer_params(
|
||||
sampling_params.vllm_xargs,
|
||||
@@ -93,6 +96,7 @@ pub(super) fn prepare_generate_request(
|
||||
include_continuous_usage,
|
||||
include_logprobs,
|
||||
include_prompt_logprobs,
|
||||
return_token_ids,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -46,6 +46,7 @@ pub struct GenerateRequest {
|
||||
pub ec_transfer_params: Option<HashMap<String, Value>>,
|
||||
/// Raw multimodal input; server resolves media. Mutually exclusive with `features`.
|
||||
pub content_parts: Option<Vec<MediaContentPart>>,
|
||||
pub return_token_ids: Option<bool>,
|
||||
#[serde(flatten)]
|
||||
pub other: Map<String, Value>,
|
||||
}
|
||||
@@ -81,6 +82,8 @@ pub(super) struct GenerateStreamResponse {
|
||||
pub request_id: String,
|
||||
pub choices: Vec<GenerateResponseStreamChoice>,
|
||||
pub usage: Option<Usage>,
|
||||
pub prompt_token_ids: Option<Vec<u32>>,
|
||||
pub mm_placeholders: Option<MultiModalPlaceholders>,
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `GenerateResponse` class.
|
||||
@@ -89,10 +92,20 @@ pub(super) struct GenerateResponse {
|
||||
pub request_id: String,
|
||||
pub choices: Vec<GenerateResponseChoice>,
|
||||
pub prompt_logprobs: Option<Vec<Option<HashMap<u32, GenerateLogprob>>>>,
|
||||
pub prompt_token_ids: Option<Vec<u32>>,
|
||||
pub mm_placeholders: Option<MultiModalPlaceholders>,
|
||||
pub kv_transfer_params: Option<Value>,
|
||||
pub ec_transfer_params: Option<Value>,
|
||||
}
|
||||
|
||||
pub(super) type MultiModalPlaceholders = HashMap<String, Vec<PlaceholderRangeInfo>>;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub(super) struct PlaceholderRangeInfo {
|
||||
pub offset: usize,
|
||||
pub length: usize,
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `Logprob` class used in prompt-logprobs payloads.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct GenerateLogprob {
|
||||
|
||||
@@ -105,6 +105,7 @@ fn lower_render_request(
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
content_parts: None,
|
||||
return_token_ids: None,
|
||||
other: Default::default(),
|
||||
};
|
||||
validate_generate_request(&request, &state.served_model_names)?;
|
||||
|
||||
Reference in New Issue
Block a user