[Frontend] Add content_parts to /inference/v1/generate for raw multim… (#51478)
Signed-off-by: Allen Shen <aoshen@inferact.ai> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
1a1727330a
commit
33584901ba
@@ -45,14 +45,31 @@ use crate::utils::resolve_request_context;
|
||||
pub async fn generate(
|
||||
State(state): State<Arc<AppState>>,
|
||||
headers: HeaderMap,
|
||||
ValidatedJson(body): ValidatedJson<GenerateRequest>,
|
||||
ValidatedJson(mut body): ValidatedJson<GenerateRequest>,
|
||||
) -> Response {
|
||||
let request_context = resolve_request_context(&headers, body.request_id.as_deref());
|
||||
let lora_resolution = state.resolve_model_with_loras(body.model.as_deref()).await;
|
||||
let prepared = match prepare_generate_request(body, &lora_resolution, request_context) {
|
||||
Ok(prepared) => prepared,
|
||||
Err(error) => return error.into_response(),
|
||||
|
||||
let mm_features = if let Some(parts) = body.content_parts.take() {
|
||||
match state.chat.prepare_media(parts, &mut body.token_ids).await {
|
||||
Ok(features) => features,
|
||||
Err(e) => {
|
||||
return ApiError::invalid_request(
|
||||
format!("failed to resolve content_parts: {}", e.as_report()),
|
||||
Some("content_parts"),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let prepared =
|
||||
match prepare_generate_request(body, &lora_resolution, request_context, mm_features) {
|
||||
Ok(prepared) => prepared,
|
||||
Err(error) => return error.into_response(),
|
||||
};
|
||||
let request_span = tracing::info_span!(
|
||||
"generate",
|
||||
request_id = %prepared.request_id,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use vllm_engine_core_client::protocol::multimodal::MmFeatures;
|
||||
use vllm_text::{Prompt, TextDecodeOptions, TextRequest};
|
||||
|
||||
use super::types::GenerateRequest;
|
||||
@@ -37,6 +38,7 @@ pub(super) fn prepare_generate_request(
|
||||
request: GenerateRequest,
|
||||
lora_resolution: &LoraModelResolution,
|
||||
ctx: ResolvedRequestContext,
|
||||
mm_features: Option<MmFeatures>,
|
||||
) -> Result<PreparedRequest, ApiError> {
|
||||
validate::validate_request_compat(&request, &lora_resolution.model_names)?;
|
||||
|
||||
@@ -67,7 +69,7 @@ pub(super) fn prepare_generate_request(
|
||||
let text_request = TextRequest {
|
||||
request_id: ctx.request_id.clone(),
|
||||
prompt: Prompt::TokenIds(request.token_ids),
|
||||
mm_features: None,
|
||||
mm_features,
|
||||
sampling_params,
|
||||
decode_options: TextDecodeOptions::default(),
|
||||
intermediate: false,
|
||||
@@ -134,6 +136,7 @@ mod tests {
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
None,
|
||||
)
|
||||
.expect("prepare");
|
||||
|
||||
@@ -175,6 +178,7 @@ mod tests {
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
None,
|
||||
)
|
||||
.expect("prepare");
|
||||
|
||||
@@ -204,6 +208,7 @@ mod tests {
|
||||
request,
|
||||
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
|
||||
ResolvedRequestContext::default(),
|
||||
None,
|
||||
)
|
||||
.expect("prepare");
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use llm_multimodal::MediaContentPart;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
use validator::Validate;
|
||||
@@ -26,6 +27,8 @@ pub struct GenerateRequest {
|
||||
pub priority: i32,
|
||||
pub kv_transfer_params: Option<HashMap<String, Value>>,
|
||||
pub ec_transfer_params: Option<HashMap<String, Value>>,
|
||||
/// Raw multimodal input; server resolves media. Mutually exclusive with `features`.
|
||||
pub content_parts: Option<Vec<MediaContentPart>>,
|
||||
#[serde(flatten)]
|
||||
pub other: Map<String, Value>,
|
||||
}
|
||||
|
||||
@@ -93,6 +93,7 @@ fn lower_render_request(
|
||||
priority: text_request.priority,
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
content_parts: None,
|
||||
other: Default::default(),
|
||||
};
|
||||
validate_generate_request(&request, &state.served_model_names)?;
|
||||
|
||||
@@ -156,3 +156,87 @@ async def test_render_to_generate_roundtrip(client, test_image):
|
||||
assert "red" in text.lower(), (
|
||||
f"Expected model to identify the red image, got: {text!r}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_content_parts_generates_tokens(client, test_image):
|
||||
"""content_parts with raw media should produce output tokens."""
|
||||
data_url = encode_image_url(test_image, format="PNG")
|
||||
|
||||
render_resp = await client.post(
|
||||
RENDER_ENDPOINT,
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": data_url}},
|
||||
{"type": "text", "text": "What color is this?"},
|
||||
],
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
render_resp.raise_for_status()
|
||||
token_ids = render_resp.json()["token_ids"]
|
||||
|
||||
gen_resp = await client.post(
|
||||
GEN_ENDPOINT,
|
||||
json={
|
||||
"token_ids": token_ids,
|
||||
"content_parts": [{"type": "image_url", "url": data_url}],
|
||||
"sampling_params": {"max_tokens": 10, "temperature": 0.0},
|
||||
},
|
||||
)
|
||||
gen_resp.raise_for_status()
|
||||
gen_data = gen_resp.json()
|
||||
|
||||
assert "choices" in gen_data
|
||||
choice = gen_data["choices"][0]
|
||||
assert "token_ids" in choice
|
||||
assert len(choice["token_ids"]) > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_content_parts_streaming(client, test_image):
|
||||
"""content_parts should work with streaming."""
|
||||
import json as json_mod
|
||||
|
||||
data_url = encode_image_url(test_image, format="PNG")
|
||||
|
||||
render_resp = await client.post(
|
||||
RENDER_ENDPOINT,
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": data_url}},
|
||||
{"type": "text", "text": "Describe this."},
|
||||
],
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
render_resp.raise_for_status()
|
||||
token_ids = render_resp.json()["token_ids"]
|
||||
|
||||
async with client.stream(
|
||||
"POST",
|
||||
GEN_ENDPOINT,
|
||||
json={
|
||||
"token_ids": token_ids,
|
||||
"content_parts": [{"type": "image_url", "url": data_url}],
|
||||
"sampling_params": {"max_tokens": 8, "temperature": 0.0},
|
||||
"stream": True,
|
||||
},
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
chunks = []
|
||||
async for line in resp.aiter_lines():
|
||||
if line.startswith("data: ") and line != "data: [DONE]":
|
||||
chunks.append(json_mod.loads(line[6:]))
|
||||
|
||||
assert len(chunks) > 0
|
||||
|
||||
@@ -104,6 +104,16 @@ class GenerateRequest(BaseModel):
|
||||
features: MultiModalFeatures | None = None
|
||||
"""Multimodal hashes and placeholder positions (populated for MM inputs)."""
|
||||
|
||||
content_parts: list[dict[str, Any]] | None = None
|
||||
"""Raw multimodal input; server resolves media. Mutually exclusive
|
||||
with ``features``."""
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _check_mm_fields_exclusive(self) -> "GenerateRequest":
|
||||
if self.content_parts and self.features:
|
||||
raise ValueError("content_parts and features are mutually exclusive")
|
||||
return self
|
||||
|
||||
sampling_params: SamplingParams
|
||||
"""The sampling parameters for the model."""
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ import pybase64 as base64
|
||||
from fastapi import Request
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.chat_utils import AsyncMultiModalItemTracker
|
||||
from vllm.entrypoints.generate.base.serving import (
|
||||
GenerateBaseServing,
|
||||
clamp_prompt_logprobs,
|
||||
@@ -33,7 +34,7 @@ from vllm.entrypoints.openai.engine.protocol import (
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.inputs import EngineInput, mm_input
|
||||
from vllm.inputs import EngineInput, TokensPrompt, mm_input
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.multimodal.inputs import (
|
||||
@@ -143,7 +144,29 @@ class ServingTokens(GenerateBaseServing):
|
||||
return self.create_error_response(e)
|
||||
|
||||
engine_input: EngineInput
|
||||
if features := request.features:
|
||||
if request.content_parts:
|
||||
tracker = AsyncMultiModalItemTracker(self.model_config)
|
||||
mm_parser = tracker.create_parser()
|
||||
for part in request.content_parts:
|
||||
ptype = part.get("type", "")
|
||||
url = part.get("url")
|
||||
uuid = part.get("uuid")
|
||||
if ptype == "image_url":
|
||||
mm_parser.parse_image(url, uuid)
|
||||
elif ptype == "audio_url":
|
||||
mm_parser.parse_audio(url, uuid)
|
||||
elif ptype == "video_url":
|
||||
mm_parser.parse_video(url, uuid)
|
||||
mm_data, mm_uuids = await tracker.resolve_items()
|
||||
prompt = TokensPrompt(prompt_token_ids=request.token_ids)
|
||||
if mm_data:
|
||||
prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids:
|
||||
prompt["multi_modal_uuids"] = mm_uuids
|
||||
(engine_input,) = await self.online_renderer.renderer.render_cmpl_async(
|
||||
[prompt]
|
||||
)
|
||||
elif features := request.features:
|
||||
# Convert PlaceholderRangeInfo → PlaceholderRange per modality.
|
||||
mm_placeholders: dict[str, list[PlaceholderRange]] = {
|
||||
modality: [
|
||||
|
||||
Reference in New Issue
Block a user