[Frontend] Add content_parts to /inference/v1/generate for raw multim… (#51478)
Signed-off-by: Allen Shen <aoshen@inferact.ai> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
1a1727330a
commit
33584901ba
@@ -45,14 +45,31 @@ use crate::utils::resolve_request_context;
|
||||
pub async fn generate(
|
||||
State(state): State<Arc<AppState>>,
|
||||
headers: HeaderMap,
|
||||
ValidatedJson(body): ValidatedJson<GenerateRequest>,
|
||||
ValidatedJson(mut body): ValidatedJson<GenerateRequest>,
|
||||
) -> Response {
|
||||
let request_context = resolve_request_context(&headers, body.request_id.as_deref());
|
||||
let lora_resolution = state.resolve_model_with_loras(body.model.as_deref()).await;
|
||||
let prepared = match prepare_generate_request(body, &lora_resolution, request_context) {
|
||||
Ok(prepared) => prepared,
|
||||
Err(error) => return error.into_response(),
|
||||
|
||||
let mm_features = if let Some(parts) = body.content_parts.take() {
|
||||
match state.chat.prepare_media(parts, &mut body.token_ids).await {
|
||||
Ok(features) => features,
|
||||
Err(e) => {
|
||||
return ApiError::invalid_request(
|
||||
format!("failed to resolve content_parts: {}", e.as_report()),
|
||||
Some("content_parts"),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let prepared =
|
||||
match prepare_generate_request(body, &lora_resolution, request_context, mm_features) {
|
||||
Ok(prepared) => prepared,
|
||||
Err(error) => return error.into_response(),
|
||||
};
|
||||
let request_span = tracing::info_span!(
|
||||
"generate",
|
||||
request_id = %prepared.request_id,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use vllm_engine_core_client::protocol::multimodal::MmFeatures;
|
||||
use vllm_text::{Prompt, TextDecodeOptions, TextRequest};
|
||||
|
||||
use super::types::GenerateRequest;
|
||||
@@ -37,6 +38,7 @@ pub(super) fn prepare_generate_request(
|
||||
request: GenerateRequest,
|
||||
lora_resolution: &LoraModelResolution,
|
||||
ctx: ResolvedRequestContext,
|
||||
mm_features: Option<MmFeatures>,
|
||||
) -> Result<PreparedRequest, ApiError> {
|
||||
validate::validate_request_compat(&request, &lora_resolution.model_names)?;
|
||||
|
||||
@@ -67,7 +69,7 @@ pub(super) fn prepare_generate_request(
|
||||
let text_request = TextRequest {
|
||||
request_id: ctx.request_id.clone(),
|
||||
prompt: Prompt::TokenIds(request.token_ids),
|
||||
mm_features: None,
|
||||
mm_features,
|
||||
sampling_params,
|
||||
decode_options: TextDecodeOptions::default(),
|
||||
intermediate: false,
|
||||
@@ -134,6 +136,7 @@ mod tests {
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
None,
|
||||
)
|
||||
.expect("prepare");
|
||||
|
||||
@@ -175,6 +178,7 @@ mod tests {
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
None,
|
||||
)
|
||||
.expect("prepare");
|
||||
|
||||
@@ -204,6 +208,7 @@ mod tests {
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
None,
|
||||
)
|
||||
.expect("prepare");
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use llm_multimodal::MediaContentPart;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
use validator::Validate;
|
||||
@@ -26,6 +27,8 @@ pub struct GenerateRequest {
|
||||
pub priority: i32,
|
||||
pub kv_transfer_params: Option<HashMap<String, Value>>,
|
||||
pub ec_transfer_params: Option<HashMap<String, Value>>,
|
||||
/// Raw multimodal input; server resolves media. Mutually exclusive with `features`.
|
||||
pub content_parts: Option<Vec<MediaContentPart>>,
|
||||
#[serde(flatten)]
|
||||
pub other: Map<String, Value>,
|
||||
}
|
||||
|
||||
@@ -93,6 +93,7 @@ fn lower_render_request(
|
||||
priority: text_request.priority,
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
content_parts: None,
|
||||
other: Default::default(),
|
||||
};
|
||||
validate_generate_request(&request, &state.served_model_names)?;
|
||||
|
||||
Reference in New Issue
Block a user