[Frontend] Add content_parts to /inference/v1/generate for raw multim… (#51478)

Signed-off-by: Allen Shen <aoshen@inferact.ai>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
aoshen02
2026-08-11 10:00:12 +08:00
committed by GitHub
co-authored by Claude Opus 4.6 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 1a1727330a
commit 33584901ba
7 changed files with 150 additions and 7 deletions
@@ -45,14 +45,31 @@ use crate::utils::resolve_request_context;
pub async fn generate(
State(state): State<Arc<AppState>>,
headers: HeaderMap,
ValidatedJson(body): ValidatedJson<GenerateRequest>,
ValidatedJson(mut body): ValidatedJson<GenerateRequest>,
) -> Response {
let request_context = resolve_request_context(&headers, body.request_id.as_deref());
let lora_resolution = state.resolve_model_with_loras(body.model.as_deref()).await;
let prepared = match prepare_generate_request(body, &lora_resolution, request_context) {
Ok(prepared) => prepared,
Err(error) => return error.into_response(),
let mm_features = if let Some(parts) = body.content_parts.take() {
match state.chat.prepare_media(parts, &mut body.token_ids).await {
Ok(features) => features,
Err(e) => {
return ApiError::invalid_request(
format!("failed to resolve content_parts: {}", e.as_report()),
Some("content_parts"),
)
.into_response();
}
}
} else {
None
};
let prepared =
match prepare_generate_request(body, &lora_resolution, request_context, mm_features) {
Ok(prepared) => prepared,
Err(error) => return error.into_response(),
};
let request_span = tracing::info_span!(
"generate",
request_id = %prepared.request_id,
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use vllm_engine_core_client::protocol::multimodal::MmFeatures;
use vllm_text::{Prompt, TextDecodeOptions, TextRequest};
use super::types::GenerateRequest;
@@ -37,6 +38,7 @@ pub(super) fn prepare_generate_request(
request: GenerateRequest,
lora_resolution: &LoraModelResolution,
ctx: ResolvedRequestContext,
mm_features: Option<MmFeatures>,
) -> Result<PreparedRequest, ApiError> {
validate::validate_request_compat(&request, &lora_resolution.model_names)?;
@@ -67,7 +69,7 @@ pub(super) fn prepare_generate_request(
let text_request = TextRequest {
request_id: ctx.request_id.clone(),
prompt: Prompt::TokenIds(request.token_ids),
mm_features: None,
mm_features,
sampling_params,
decode_options: TextDecodeOptions::default(),
intermediate: false,
@@ -134,6 +136,7 @@ mod tests {
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
None,
)
.expect("prepare");
@@ -175,6 +178,7 @@ mod tests {
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
None,
)
.expect("prepare");
@@ -204,6 +208,7 @@ mod tests {
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
None,
)
.expect("prepare");
@@ -3,6 +3,7 @@
use std::collections::HashMap;
use llm_multimodal::MediaContentPart;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use validator::Validate;
@@ -26,6 +27,8 @@ pub struct GenerateRequest {
pub priority: i32,
pub kv_transfer_params: Option<HashMap<String, Value>>,
pub ec_transfer_params: Option<HashMap<String, Value>>,
/// Raw multimodal input; server resolves media. Mutually exclusive with `features`.
pub content_parts: Option<Vec<MediaContentPart>>,
#[serde(flatten)]
pub other: Map<String, Value>,
}
+1
View File
@@ -93,6 +93,7 @@ fn lower_render_request(
priority: text_request.priority,
kv_transfer_params: None,
ec_transfer_params: None,
content_parts: None,
other: Default::default(),
};
validate_generate_request(&request, &state.served_model_names)?;