[Frontend] Add content_parts to /inference/v1/generate for raw multim… (#51478)

Signed-off-by: Allen Shen <aoshen@inferact.ai>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
aoshen02
2026-08-11 10:00:12 +08:00
committed by GitHub
co-authored by Claude Opus 4.6 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 1a1727330a
commit 33584901ba
7 changed files with 150 additions and 7 deletions
@@ -45,14 +45,31 @@ use crate::utils::resolve_request_context;
pub async fn generate(
State(state): State<Arc<AppState>>,
headers: HeaderMap,
ValidatedJson(body): ValidatedJson<GenerateRequest>,
ValidatedJson(mut body): ValidatedJson<GenerateRequest>,
) -> Response {
let request_context = resolve_request_context(&headers, body.request_id.as_deref());
let lora_resolution = state.resolve_model_with_loras(body.model.as_deref()).await;
let prepared = match prepare_generate_request(body, &lora_resolution, request_context) {
Ok(prepared) => prepared,
Err(error) => return error.into_response(),
let mm_features = if let Some(parts) = body.content_parts.take() {
match state.chat.prepare_media(parts, &mut body.token_ids).await {
Ok(features) => features,
Err(e) => {
return ApiError::invalid_request(
format!("failed to resolve content_parts: {}", e.as_report()),
Some("content_parts"),
)
.into_response();
}
}
} else {
None
};
let prepared =
match prepare_generate_request(body, &lora_resolution, request_context, mm_features) {
Ok(prepared) => prepared,
Err(error) => return error.into_response(),
};
let request_span = tracing::info_span!(
"generate",
request_id = %prepared.request_id,
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use vllm_engine_core_client::protocol::multimodal::MmFeatures;
use vllm_text::{Prompt, TextDecodeOptions, TextRequest};
use super::types::GenerateRequest;
@@ -37,6 +38,7 @@ pub(super) fn prepare_generate_request(
request: GenerateRequest,
lora_resolution: &LoraModelResolution,
ctx: ResolvedRequestContext,
mm_features: Option<MmFeatures>,
) -> Result<PreparedRequest, ApiError> {
validate::validate_request_compat(&request, &lora_resolution.model_names)?;
@@ -67,7 +69,7 @@ pub(super) fn prepare_generate_request(
let text_request = TextRequest {
request_id: ctx.request_id.clone(),
prompt: Prompt::TokenIds(request.token_ids),
mm_features: None,
mm_features,
sampling_params,
decode_options: TextDecodeOptions::default(),
intermediate: false,
@@ -134,6 +136,7 @@ mod tests {
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
None,
)
.expect("prepare");
@@ -175,6 +178,7 @@ mod tests {
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
None,
)
.expect("prepare");
@@ -204,6 +208,7 @@ mod tests {
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
None,
)
.expect("prepare");
@@ -3,6 +3,7 @@
use std::collections::HashMap;
use llm_multimodal::MediaContentPart;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use validator::Validate;
@@ -26,6 +27,8 @@ pub struct GenerateRequest {
pub priority: i32,
pub kv_transfer_params: Option<HashMap<String, Value>>,
pub ec_transfer_params: Option<HashMap<String, Value>>,
/// Raw multimodal input; server resolves media. Mutually exclusive with `features`.
pub content_parts: Option<Vec<MediaContentPart>>,
#[serde(flatten)]
pub other: Map<String, Value>,
}
+1
View File
@@ -93,6 +93,7 @@ fn lower_render_request(
priority: text_request.priority,
kv_transfer_params: None,
ec_transfer_params: None,
content_parts: None,
other: Default::default(),
};
validate_generate_request(&request, &state.served_model_names)?;
@@ -156,3 +156,87 @@ async def test_render_to_generate_roundtrip(client, test_image):
assert "red" in text.lower(), (
f"Expected model to identify the red image, got: {text!r}"
)
@pytest.mark.asyncio
async def test_content_parts_generates_tokens(client, test_image):
"""content_parts with raw media should produce output tokens."""
data_url = encode_image_url(test_image, format="PNG")
render_resp = await client.post(
RENDER_ENDPOINT,
json={
"model": MODEL_NAME,
"messages": [
{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": data_url}},
{"type": "text", "text": "What color is this?"},
],
}
],
},
)
render_resp.raise_for_status()
token_ids = render_resp.json()["token_ids"]
gen_resp = await client.post(
GEN_ENDPOINT,
json={
"token_ids": token_ids,
"content_parts": [{"type": "image_url", "url": data_url}],
"sampling_params": {"max_tokens": 10, "temperature": 0.0},
},
)
gen_resp.raise_for_status()
gen_data = gen_resp.json()
assert "choices" in gen_data
choice = gen_data["choices"][0]
assert "token_ids" in choice
assert len(choice["token_ids"]) > 0
@pytest.mark.asyncio
async def test_content_parts_streaming(client, test_image):
"""content_parts should work with streaming."""
import json as json_mod
data_url = encode_image_url(test_image, format="PNG")
render_resp = await client.post(
RENDER_ENDPOINT,
json={
"model": MODEL_NAME,
"messages": [
{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": data_url}},
{"type": "text", "text": "Describe this."},
],
}
],
},
)
render_resp.raise_for_status()
token_ids = render_resp.json()["token_ids"]
async with client.stream(
"POST",
GEN_ENDPOINT,
json={
"token_ids": token_ids,
"content_parts": [{"type": "image_url", "url": data_url}],
"sampling_params": {"max_tokens": 8, "temperature": 0.0},
"stream": True,
},
) as resp:
resp.raise_for_status()
chunks = []
async for line in resp.aiter_lines():
if line.startswith("data: ") and line != "data: [DONE]":
chunks.append(json_mod.loads(line[6:]))
assert len(chunks) > 0
@@ -104,6 +104,16 @@ class GenerateRequest(BaseModel):
features: MultiModalFeatures | None = None
"""Multimodal hashes and placeholder positions (populated for MM inputs)."""
content_parts: list[dict[str, Any]] | None = None
"""Raw multimodal input; server resolves media. Mutually exclusive
with ``features``."""
@model_validator(mode="after")
def _check_mm_fields_exclusive(self) -> "GenerateRequest":
if self.content_parts and self.features:
raise ValueError("content_parts and features are mutually exclusive")
return self
sampling_params: SamplingParams
"""The sampling parameters for the model."""
@@ -14,6 +14,7 @@ import pybase64 as base64
from fastapi import Request
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.chat_utils import AsyncMultiModalItemTracker
from vllm.entrypoints.generate.base.serving import (
GenerateBaseServing,
clamp_prompt_logprobs,
@@ -33,7 +34,7 @@ from vllm.entrypoints.openai.engine.protocol import (
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.inputs import EngineInput, mm_input
from vllm.inputs import EngineInput, TokensPrompt, mm_input
from vllm.logger import init_logger
from vllm.logprobs import Logprob
from vllm.multimodal.inputs import (
@@ -143,7 +144,29 @@ class ServingTokens(GenerateBaseServing):
return self.create_error_response(e)
engine_input: EngineInput
if features := request.features:
if request.content_parts:
tracker = AsyncMultiModalItemTracker(self.model_config)
mm_parser = tracker.create_parser()
for part in request.content_parts:
ptype = part.get("type", "")
url = part.get("url")
uuid = part.get("uuid")
if ptype == "image_url":
mm_parser.parse_image(url, uuid)
elif ptype == "audio_url":
mm_parser.parse_audio(url, uuid)
elif ptype == "video_url":
mm_parser.parse_video(url, uuid)
mm_data, mm_uuids = await tracker.resolve_items()
prompt = TokensPrompt(prompt_token_ids=request.token_ids)
if mm_data:
prompt["multi_modal_data"] = mm_data
if mm_uuids:
prompt["multi_modal_uuids"] = mm_uuids
(engine_input,) = await self.online_renderer.renderer.render_cmpl_async(
[prompt]
)
elif features := request.features:
# Convert PlaceholderRangeInfo → PlaceholderRange per modality.
mm_placeholders: dict[str, list[PlaceholderRange]] = {
modality: [