diff --git a/rust/src/server/src/routes/inference/generate.rs b/rust/src/server/src/routes/inference/generate.rs index 57d2724cb6..61fed421d3 100644 --- a/rust/src/server/src/routes/inference/generate.rs +++ b/rust/src/server/src/routes/inference/generate.rs @@ -45,14 +45,31 @@ use crate::utils::resolve_request_context; pub async fn generate( State(state): State>, headers: HeaderMap, - ValidatedJson(body): ValidatedJson, + ValidatedJson(mut body): ValidatedJson, ) -> Response { let request_context = resolve_request_context(&headers, body.request_id.as_deref()); let lora_resolution = state.resolve_model_with_loras(body.model.as_deref()).await; - let prepared = match prepare_generate_request(body, &lora_resolution, request_context) { - Ok(prepared) => prepared, - Err(error) => return error.into_response(), + + let mm_features = if let Some(parts) = body.content_parts.take() { + match state.chat.prepare_media(parts, &mut body.token_ids).await { + Ok(features) => features, + Err(e) => { + return ApiError::invalid_request( + format!("failed to resolve content_parts: {}", e.as_report()), + Some("content_parts"), + ) + .into_response(); + } + } + } else { + None }; + + let prepared = + match prepare_generate_request(body, &lora_resolution, request_context, mm_features) { + Ok(prepared) => prepared, + Err(error) => return error.into_response(), + }; let request_span = tracing::info_span!( "generate", request_id = %prepared.request_id, diff --git a/rust/src/server/src/routes/inference/generate/convert.rs b/rust/src/server/src/routes/inference/generate/convert.rs index 10361e679a..5c14107989 100644 --- a/rust/src/server/src/routes/inference/generate/convert.rs +++ b/rust/src/server/src/routes/inference/generate/convert.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project +use vllm_engine_core_client::protocol::multimodal::MmFeatures; use vllm_text::{Prompt, TextDecodeOptions, TextRequest}; use super::types::GenerateRequest; @@ -37,6 +38,7 @@ pub(super) fn prepare_generate_request( request: GenerateRequest, lora_resolution: &LoraModelResolution, ctx: ResolvedRequestContext, + mm_features: Option, ) -> Result { validate::validate_request_compat(&request, &lora_resolution.model_names)?; @@ -67,7 +69,7 @@ pub(super) fn prepare_generate_request( let text_request = TextRequest { request_id: ctx.request_id.clone(), prompt: Prompt::TokenIds(request.token_ids), - mm_features: None, + mm_features, sampling_params, decode_options: TextDecodeOptions::default(), intermediate: false, @@ -134,6 +136,7 @@ mod tests { request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]), ResolvedRequestContext::default(), + None, ) .expect("prepare"); @@ -175,6 +178,7 @@ mod tests { request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]), ResolvedRequestContext::default(), + None, ) .expect("prepare"); @@ -204,6 +208,7 @@ mod tests { request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]), ResolvedRequestContext::default(), + None, ) .expect("prepare"); diff --git a/rust/src/server/src/routes/inference/generate/types.rs b/rust/src/server/src/routes/inference/generate/types.rs index 18095d5288..200209e095 100644 --- a/rust/src/server/src/routes/inference/generate/types.rs +++ b/rust/src/server/src/routes/inference/generate/types.rs @@ -3,6 +3,7 @@ use std::collections::HashMap; +use llm_multimodal::MediaContentPart; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use validator::Validate; @@ -26,6 +27,8 @@ pub struct GenerateRequest { pub priority: i32, pub kv_transfer_params: Option>, pub ec_transfer_params: Option>, + /// Raw multimodal input; server resolves media. Mutually exclusive with `features`. + pub content_parts: Option>, #[serde(flatten)] pub other: Map, } diff --git a/rust/src/server/src/routes/render.rs b/rust/src/server/src/routes/render.rs index e5a66cec95..1972c02d09 100644 --- a/rust/src/server/src/routes/render.rs +++ b/rust/src/server/src/routes/render.rs @@ -93,6 +93,7 @@ fn lower_render_request( priority: text_request.priority, kv_transfer_params: None, ec_transfer_params: None, + content_parts: None, other: Default::default(), }; validate_generate_request(&request, &state.served_model_names)?; diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_serving_multimodal_tokens.py b/tests/entrypoints/scale_out/token_in_token_out/test_serving_multimodal_tokens.py index e13dd425af..b4811666d4 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_serving_multimodal_tokens.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_serving_multimodal_tokens.py @@ -156,3 +156,87 @@ async def test_render_to_generate_roundtrip(client, test_image): assert "red" in text.lower(), ( f"Expected model to identify the red image, got: {text!r}" ) + + +@pytest.mark.asyncio +async def test_content_parts_generates_tokens(client, test_image): + """content_parts with raw media should produce output tokens.""" + data_url = encode_image_url(test_image, format="PNG") + + render_resp = await client.post( + RENDER_ENDPOINT, + json={ + "model": MODEL_NAME, + "messages": [ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": data_url}}, + {"type": "text", "text": "What color is this?"}, + ], + } + ], + }, + ) + render_resp.raise_for_status() + token_ids = render_resp.json()["token_ids"] + + gen_resp = await client.post( + GEN_ENDPOINT, + json={ + "token_ids": token_ids, + "content_parts": [{"type": "image_url", "url": data_url}], + "sampling_params": {"max_tokens": 10, "temperature": 0.0}, + }, + ) + gen_resp.raise_for_status() + gen_data = gen_resp.json() + + assert "choices" in gen_data + choice = gen_data["choices"][0] + assert "token_ids" in choice + assert len(choice["token_ids"]) > 0 + + +@pytest.mark.asyncio +async def test_content_parts_streaming(client, test_image): + """content_parts should work with streaming.""" + import json as json_mod + + data_url = encode_image_url(test_image, format="PNG") + + render_resp = await client.post( + RENDER_ENDPOINT, + json={ + "model": MODEL_NAME, + "messages": [ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": data_url}}, + {"type": "text", "text": "Describe this."}, + ], + } + ], + }, + ) + render_resp.raise_for_status() + token_ids = render_resp.json()["token_ids"] + + async with client.stream( + "POST", + GEN_ENDPOINT, + json={ + "token_ids": token_ids, + "content_parts": [{"type": "image_url", "url": data_url}], + "sampling_params": {"max_tokens": 8, "temperature": 0.0}, + "stream": True, + }, + ) as resp: + resp.raise_for_status() + chunks = [] + async for line in resp.aiter_lines(): + if line.startswith("data: ") and line != "data: [DONE]": + chunks.append(json_mod.loads(line[6:])) + + assert len(chunks) > 0 diff --git a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py index 86c0a8c8b1..606989c31d 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py @@ -104,6 +104,16 @@ class GenerateRequest(BaseModel): features: MultiModalFeatures | None = None """Multimodal hashes and placeholder positions (populated for MM inputs).""" + content_parts: list[dict[str, Any]] | None = None + """Raw multimodal input; server resolves media. Mutually exclusive + with ``features``.""" + + @model_validator(mode="after") + def _check_mm_fields_exclusive(self) -> "GenerateRequest": + if self.content_parts and self.features: + raise ValueError("content_parts and features are mutually exclusive") + return self + sampling_params: SamplingParams """The sampling parameters for the model.""" diff --git a/vllm/entrypoints/scale_out/token_in_token_out/serving.py b/vllm/entrypoints/scale_out/token_in_token_out/serving.py index 7b5e2bbb3d..c5b0ffea69 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/serving.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/serving.py @@ -14,6 +14,7 @@ import pybase64 as base64 from fastapi import Request from vllm.engine.protocol import EngineClient +from vllm.entrypoints.chat_utils import AsyncMultiModalItemTracker from vllm.entrypoints.generate.base.serving import ( GenerateBaseServing, clamp_prompt_logprobs, @@ -33,7 +34,7 @@ from vllm.entrypoints.openai.engine.protocol import ( from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage from vllm.entrypoints.serve.utils.request_logger import RequestLogger -from vllm.inputs import EngineInput, mm_input +from vllm.inputs import EngineInput, TokensPrompt, mm_input from vllm.logger import init_logger from vllm.logprobs import Logprob from vllm.multimodal.inputs import ( @@ -143,7 +144,29 @@ class ServingTokens(GenerateBaseServing): return self.create_error_response(e) engine_input: EngineInput - if features := request.features: + if request.content_parts: + tracker = AsyncMultiModalItemTracker(self.model_config) + mm_parser = tracker.create_parser() + for part in request.content_parts: + ptype = part.get("type", "") + url = part.get("url") + uuid = part.get("uuid") + if ptype == "image_url": + mm_parser.parse_image(url, uuid) + elif ptype == "audio_url": + mm_parser.parse_audio(url, uuid) + elif ptype == "video_url": + mm_parser.parse_video(url, uuid) + mm_data, mm_uuids = await tracker.resolve_items() + prompt = TokensPrompt(prompt_token_ids=request.token_ids) + if mm_data: + prompt["multi_modal_data"] = mm_data + if mm_uuids: + prompt["multi_modal_uuids"] = mm_uuids + (engine_input,) = await self.online_renderer.renderer.render_cmpl_async( + [prompt] + ) + elif features := request.features: # Convert PlaceholderRangeInfo → PlaceholderRange per modality. mm_placeholders: dict[str, list[PlaceholderRange]] = { modality: [