From 31759ccb6d560e150849273b47e15680a0120169 Mon Sep 17 00:00:00 2001 From: Biswa Panda Date: Fri, 11 Sep 2026 19:29:49 -0700 Subject: [PATCH] [Rust Frontend][Multimodal] Accept preprocessed multimodal gRPC features (#55047) Signed-off-by: Biswa Panda Signed-off-by: Bugen Zhao Co-authored-by: Bugen Zhao Co-authored-by: Codex --- rust/Cargo.lock | 2 +- rust/Cargo.toml | 2 +- rust/proto/Cargo.toml | 2 +- rust/proto/README.md | 2 +- rust/proto/inference.proto | 13 + rust/src/chat/src/error.rs | 3 + rust/src/chat/src/lib.rs | 19 +- rust/src/chat/src/multimodal.rs | 42 ++- rust/src/chat/src/multimodal/input.rs | 29 ++ rust/src/chat/src/multimodal/preprocessed.rs | 274 +++++++++++++++ rust/src/server/Cargo.toml | 2 +- rust/src/server/src/grpc/convert.rs | 91 ----- rust/src/server/src/grpc/inference.rs | 18 +- rust/src/server/src/grpc/media.rs | 341 +++++++++++++++++++ rust/src/server/src/grpc/mod.rs | 1 + rust/src/server/src/grpc/tests.rs | 258 +++++++++++++- 16 files changed, 986 insertions(+), 113 deletions(-) create mode 100644 rust/src/chat/src/multimodal/input.rs create mode 100644 rust/src/chat/src/multimodal/preprocessed.rs create mode 100644 rust/src/server/src/grpc/media.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index bf19fd3225..80edfe4ebc 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -5940,7 +5940,7 @@ dependencies = [ [[package]] name = "vllm-proto" -version = "0.1.0" +version = "0.2.0" dependencies = [ "prost", "prost-types", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 75104642c3..bd9d99de62 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -147,7 +147,7 @@ vllm-llm = { path = "src/llm" } vllm-managed-engine = { path = "src/managed-engine" } vllm-metrics = { path = "src/metrics" } vllm-parser = { path = "src/parser" } -vllm-proto = { version = "0.1.0", path = "proto" } +vllm-proto = { version = "0.2.0", path = "proto" } vllm-server = { path = "src/server" } vllm-text = { path = "src/text" } vllm-tokenizer = { path = "src/tokenizer" } diff --git a/rust/proto/Cargo.toml b/rust/proto/Cargo.toml index 3084c8c5b3..130fd69a52 100644 --- a/rust/proto/Cargo.toml +++ b/rust/proto/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vllm-proto" -version = "0.1.0" +version = "0.2.0" edition.workspace = true license.workspace = true description = "Protobuf messages and gRPC bindings for the vLLM Rust frontend" diff --git a/rust/proto/README.md b/rust/proto/README.md index 3c5dbe0d3a..62b134280b 100644 --- a/rust/proto/README.md +++ b/rust/proto/README.md @@ -17,7 +17,7 @@ Repository setup requires a `BUF_TOKEN` GitHub Actions secret with permission to ```toml [dependencies] -vllm-proto = "0.1" +vllm-proto = "0.2" ``` For example, import `vllm_proto::inference_client::InferenceClient` or diff --git a/rust/proto/inference.proto b/rust/proto/inference.proto index c854a9abdf..858d8a7a7e 100644 --- a/rust/proto/inference.proto +++ b/rust/proto/inference.proto @@ -228,7 +228,20 @@ message MediaItem { string url = 2; // http:// or https:// string data_uri = 3; // data: bytes raw_bytes = 4; + PreprocessedMediaFeatures features = 7; } string mime_type = 5; string uuid = 6; } + +// Engine-ready multimodal features produced by a version-compatible frontend. +// `kwargs` is the MessagePack wire representation of one MmKwargsItem. +message PreprocessedMediaFeatures { + // Required inline payload. Cache-only references are not accepted. + optional bytes kwargs = 1; + string identifier = 2; + uint64 offset = 3; + uint64 length = 4; + optional string mm_hash = 5; + repeated bool is_embed = 6; +} diff --git a/rust/src/chat/src/error.rs b/rust/src/chat/src/error.rs index e3299dfdd4..a8304f17aa 100644 --- a/rust/src/chat/src/error.rs +++ b/rust/src/chat/src/error.rs @@ -27,6 +27,8 @@ pub enum Error { UnsupportedModality { modality: String }, #[error("At most {limit} {modality}(s) may be provided in one prompt.")] MmLimitExceeded { modality: String, limit: usize }, + #[error("invalid inline multimodal features: {message}")] + InvalidPreprocessedMultimodal { message: String }, #[error("multimodal preprocessing error: {0}")] Multimodal(#[message] String), #[error("{kind} parsing is not available for model `{model_id}`")] @@ -106,6 +108,7 @@ impl Error { Self::UnsupportedMultimodalRenderer | Self::UnsupportedMultimodalContent(_) | Self::UnsupportedModality { .. } + | Self::InvalidPreprocessedMultimodal { .. } | Self::MmLimitExceeded { .. } => true, _ => false, diff --git a/rust/src/chat/src/lib.rs b/rust/src/chat/src/lib.rs index 6250330e48..1b5d5aea6f 100644 --- a/rust/src/chat/src/lib.rs +++ b/rust/src/chat/src/lib.rs @@ -130,7 +130,7 @@ impl ChatRequestProcessor { /// Prepare media for an already-tokenized request. async fn prepare_media( &self, - media: Vec, + media: multimodal::MultimodalInput, token_ids: &mut Vec, ) -> Result> { if media.is_empty() { @@ -140,8 +140,15 @@ impl ChatRequestProcessor { .backend .multimodal_model_info() .ok_or(Error::UnsupportedMultimodalRenderer)?; - let model_dtype = self.model_dtype.ok_or(Error::UnsupportedMultimodalRenderer)?; - let features = info.prepare_multimodal(media, token_ids, model_dtype).await?; + let features = match media { + multimodal::MultimodalInput::Raw(parts) => { + let model_dtype = self.model_dtype.ok_or(Error::UnsupportedMultimodalRenderer)?; + info.prepare_multimodal(parts, token_ids, model_dtype).await? + } + multimodal::MultimodalInput::Preprocessed(features) => { + info.prepare_preprocessed(features, token_ids.len())? + } + }; Ok(Some(features)) } @@ -286,12 +293,14 @@ impl ChatLlm { } /// Prepare media for an already-tokenized request. + /// Raw content is preprocessed; inline features are checked against model + /// capabilities, modality limits, and the final prompt positions. pub async fn prepare_media( &self, - media: Vec, + media: impl Into, token_ids: &mut Vec, ) -> Result> { - self.processor.prepare_media(media, token_ids).await + self.processor.prepare_media(media.into(), token_ids).await } /// Effective tool-call parser name for this model, if parsing is enabled. diff --git a/rust/src/chat/src/multimodal.rs b/rust/src/chat/src/multimodal.rs index a0743b8769..674979e6ce 100644 --- a/rust/src/chat/src/multimodal.rs +++ b/rust/src/chat/src/multimodal.rs @@ -39,11 +39,14 @@ use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest}; mod audio; mod expand; mod image; +mod input; mod item; +mod preprocessed; mod tensor; mod video; use self::expand::expand_prompt_token_ids; +pub use self::input::MultimodalInput; /// Resolved multimodal support for one loaded model. #[derive(Clone)] @@ -654,11 +657,16 @@ impl MultimodalModelInfo { /// /// Modalities without a configured count are unlimited. fn validate_mm_limits(&self, media_parts: &[MediaContentPart]) -> Result<()> { + self.validate_modality_limits(media_parts.iter().filter_map(media_part_limit_modality)) + } + + fn validate_modality_limits( + &self, + modalities: impl IntoIterator, + ) -> Result<()> { let mut counts: HashMap = HashMap::new(); - for part in media_parts { - if let Some(modality) = media_part_limit_modality(part) { - *counts.entry(modality).or_default() += 1; - } + for modality in modalities { + *counts.entry(modality).or_default() += 1; } for (modality, count) in counts { @@ -677,6 +685,32 @@ impl MultimodalModelInfo { Ok(()) } + /// Validate inline storage, batching, and placeholder ranges, then check + /// this model's supported modalities and item-count limits. + /// + /// `prompt_len` must include all expanded multimodal placeholders. + pub(crate) fn prepare_preprocessed( + &self, + mut features: MmFeatures, + prompt_len: usize, + ) -> Result { + preprocessed::validate_features(&mut features, prompt_len)?; + for feature in &features { + let supported = match feature.modality { + MmModality::Image => self.image.is_some(), + MmModality::Video => self.video.is_some(), + MmModality::Audio => self.audio.is_some(), + }; + if !supported { + return Err(Error::UnsupportedModality { + modality: feature.modality.as_str().to_owned(), + }); + } + } + self.validate_modality_limits(features.iter().map(|feature| feature.modality))?; + Ok(features) + } + /// Run media fetch, per-modality preprocessing, prompt expansion, and /// feature build. /// diff --git a/rust/src/chat/src/multimodal/input.rs b/rust/src/chat/src/multimodal/input.rs new file mode 100644 index 0000000000..53905f0a26 --- /dev/null +++ b/rust/src/chat/src/multimodal/input.rs @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +use llm_multimodal::MediaContentPart; +use vllm_engine_core_client::protocol::multimodal::MmFeatures; + +/// Media preparation stage supplied by a frontend handler. +#[derive(Debug)] +pub enum MultimodalInput { + /// Raw media to fetch and preprocess, expanding prompt placeholders as needed. + Raw(Vec), + /// Preprocessed features to validate against storage, batching, model, and prompt constraints. + Preprocessed(MmFeatures), +} + +impl MultimodalInput { + pub fn is_empty(&self) -> bool { + match self { + Self::Raw(parts) => parts.is_empty(), + Self::Preprocessed(features) => features.is_empty(), + } + } +} + +impl From> for MultimodalInput { + fn from(parts: Vec) -> Self { + Self::Raw(parts) + } +} diff --git a/rust/src/chat/src/multimodal/preprocessed.rs b/rust/src/chat/src/multimodal/preprocessed.rs new file mode 100644 index 0000000000..a2ff4687b1 --- /dev/null +++ b/rust/src/chat/src/multimodal/preprocessed.rs @@ -0,0 +1,274 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +use std::collections::HashMap; + +use vllm_engine_core_client::protocol::dtype::TensorDtype; +use vllm_engine_core_client::protocol::multimodal::{ + MmFeatureSpec, MmField, MmKwargValue, MmKwargsItem, +}; + +use crate::error::{ + Error, Result, bail_invalid_preprocessed_multimodal, invalid_preprocessed_multimodal, +}; + +const MAX_DEPTH: usize = 32; + +/// Validate inline storage, placeholder ranges, and batching compatibility, +/// then stably sort features by their prompt offsets. +/// Encoder cache identifiers are supplied by the producer, which must bind +/// them to the feature content. +pub(super) fn validate_features(features: &mut [MmFeatureSpec], prompt_len: usize) -> Result<()> { + let mut fields = HashMap::new(); + for feature in features.iter() { + if feature.identifier.is_empty() { + bail_invalid_preprocessed_multimodal!("feature identifier must be nonempty"); + } + let data = feature.data.as_ref().ok_or_else(|| { + invalid_preprocessed_multimodal!( + "inline data is required; cache-only features are unsupported" + ) + })?; + validate_kwargs(data)?; + for (key, elem) in data { + if let Some(previous) = fields.insert((feature.modality, key), &elem.field) + && !same_batching(previous, &elem.field) + { + bail_invalid_preprocessed_multimodal!( + "incompatible batching for {} field {key:?}", + feature.modality.as_str() + ); + } + } + let position = &feature.mm_position; + if position.length == 0 || position.offset.checked_add(position.length).is_none() { + bail_invalid_preprocessed_multimodal!( + "placeholder range must have a positive length and fit in usize" + ); + } + if position.offset + position.length > prompt_len { + bail_invalid_preprocessed_multimodal!( + "{} placeholder range exceeds prompt token IDs", + feature.modality.as_str() + ); + } + if let Some(mask) = &position.is_embed { + if mask.dtype != TensorDtype::Bool || mask.shape != [position.length] { + bail_invalid_preprocessed_multimodal!( + "is_embed must be a boolean tensor of shape [placeholder length]" + ); + } + mask.validate_inline() + .map_err(|message| Error::InvalidPreprocessedMultimodal { message })?; + } + } + // Mirror the Python frontend (`argsort_mm_positions`): features are + // ordered by their placeholder position in the prompt. + features.sort_by_key(|feature| feature.mm_position.offset); + Ok(()) +} + +fn validate_kwargs(kwargs: &MmKwargsItem) -> Result<()> { + if kwargs.is_empty() { + bail_invalid_preprocessed_multimodal!("kwargs must contain at least one field"); + } + for (key, elem) in kwargs { + if key.is_empty() { + bail_invalid_preprocessed_multimodal!("kwargs field name must be nonempty"); + } + let data = elem.data.as_ref().ok_or_else(|| { + invalid_preprocessed_multimodal!("field {key:?} requires inline data") + })?; + validate_value(data, 0)?; + if let MmField::Flat(field) = &elem.field { + match data { + MmKwargValue::Tensor(tensor) => { + let rank = tensor.shape.len() as i32; + if rank == 0 || field.dim < -rank || field.dim >= rank { + bail_invalid_preprocessed_multimodal!( + "flat field {key:?} dim is outside the tensor rank" + ); + } + } + MmKwargValue::List(_) if field.dim == 0 => {} + _ => bail_invalid_preprocessed_multimodal!( + "flat field {key:?} requires a tensor or a list with dim=0" + ), + } + } + } + Ok(()) +} + +fn validate_value(value: &MmKwargValue, depth: usize) -> Result<()> { + if depth > MAX_DEPTH { + bail_invalid_preprocessed_multimodal!("kwargs nesting exceeds {MAX_DEPTH} levels"); + } + match value { + MmKwargValue::Tensor(tensor) => tensor + .validate_inline() + .map_err(|message| Error::InvalidPreprocessedMultimodal { message }), + MmKwargValue::List(values) => { + for value in values { + validate_value(value, depth + 1)?; + } + Ok(()) + } + MmKwargValue::Int(_) | MmKwargValue::Float(_) => Ok(()), + } +} + +fn same_batching(left: &MmField, right: &MmField) -> bool { + // Slices and batch_size describe the producer's original batch. Python + // reduce_data combines the already-split item data without using them. + match (left, right) { + (MmField::Batched(a), MmField::Batched(b)) => a == b, + (MmField::Flat(a), MmField::Flat(b)) => a.dim == b.dim && a.keep_on_cpu == b.keep_on_cpu, + (MmField::Shared(a), MmField::Shared(b)) => a.keep_on_cpu == b.keep_on_cpu, + _ => false, + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use vllm_engine_core_client::protocol::multimodal::{ + MmBatchedField, MmFieldElem, MmFlatField, MmModality, MmSharedField, MmSlice, + PlaceholderRange, SliceSpec, + }; + use vllm_engine_core_client::protocol::tensor::{WireArrayData, WireTensor}; + + use super::*; + use crate::multimodal::tests::qwen3_vl_info; + + fn feature(offset: usize, field: MmField) -> MmFeatureSpec { + MmFeatureSpec { + data: Some(BTreeMap::from([( + "pixels".to_owned(), + MmFieldElem { + data: Some(MmKwargValue::Tensor( + WireTensor::from_f32(vec![2], vec![1., 2.]).unwrap(), + )), + field, + }, + )])), + modality: MmModality::Image, + identifier: format!("image-{offset}"), + mm_position: PlaceholderRange { + offset, + length: 2, + is_embed: None, + }, + mm_hash: Some("processor-hash".to_owned()), + } + } + + fn batched() -> MmField { + MmField::Batched(MmBatchedField { keep_on_cpu: false }) + } + + #[test] + fn inline_features_preserve_overlaps_and_stably_sort_prompt_positions() { + let info = qwen3_vl_info(); + let mut a = feature(2, batched()); + a.mm_position.is_embed = Some(WireTensor::from_bool(vec![2], vec![true, false]).unwrap()); + let b = feature(1, batched()); + let mut c = feature(2, batched()); + c.modality = MmModality::Video; + let expected = vec![b.clone(), a.clone(), c.clone()]; + let features = info.prepare_preprocessed(vec![a, b, c], 4).unwrap(); + assert_eq!(features, expected); + } + + #[test] + fn independently_preprocessed_items_keep_their_original_batch_metadata() { + let info = qwen3_vl_info(); + let flat = |length| { + MmField::Flat(MmFlatField { + slices: vec![MmSlice::Slice(SliceSpec { + start: Some(0), + stop: Some(length), + step: None, + })], + dim: 0, + keep_on_cpu: false, + }) + }; + let shared = |batch_size| { + MmField::Shared(MmSharedField { + batch_size, + keep_on_cpu: true, + }) + }; + for (a, b) in [(flat(2), flat(7)), (shared(1), shared(3))] { + let features = vec![feature(0, a), feature(2, b)]; + let result = info.prepare_preprocessed(features.clone(), 4).unwrap(); + assert_eq!(result, features); + } + } + + #[test] + fn inline_features_reject_cache_references_and_invalid_tensor_storage() { + let info = qwen3_vl_info(); + let original = feature(0, batched()); + let mut cached = original.clone(); + cached.data = None; + assert!(info.prepare_preprocessed(vec![cached], 4).is_err()); + let mut field_cached = original.clone(); + field_cached.data.as_mut().unwrap().get_mut("pixels").unwrap().data = None; + assert!(info.prepare_preprocessed(vec![field_cached], 4).is_err()); + for tensor in [ + WireTensor { + dtype: TensorDtype::F32, + shape: vec![2], + data: WireArrayData::AuxIndex(1), + }, + WireTensor::from_raw(TensorDtype::F32, vec![2], vec![0; 4]), + WireTensor::from_raw(TensorDtype::F32, vec![usize::MAX, 2], vec![]), + ] { + let mut item = original.clone(); + item.data.as_mut().unwrap().get_mut("pixels").unwrap().data = + Some(MmKwargValue::Tensor(tensor)); + assert!(info.prepare_preprocessed(vec![item], 4).is_err()); + } + } + + #[test] + fn inline_features_validate_placeholder_and_mask_bounds() { + let info = qwen3_vl_info(); + for (offset, length) in [(usize::MAX, 2), (0, 0)] { + let mut item = feature(offset, batched()); + item.mm_position.length = length; + assert!(info.prepare_preprocessed(vec![item], 4).is_err()); + } + assert!(info.prepare_preprocessed(vec![feature(2, batched())], 3).is_err()); + for mask in [ + WireTensor::from_bool(vec![1], vec![true]).unwrap(), + WireTensor::from_u32(vec![2], vec![1, 0]).unwrap(), + ] { + let mut item = feature(0, batched()); + item.mm_position.is_embed = Some(mask); + assert!(info.prepare_preprocessed(vec![item], 4).is_err()); + } + } + + #[test] + fn inline_features_reject_incompatible_batching_and_flat_axes() { + let info = qwen3_vl_info(); + let flat = MmField::Flat(MmFlatField { + slices: vec![], + dim: 2, + keep_on_cpu: false, + }); + assert!(info.prepare_preprocessed(vec![feature(0, flat)], 4).is_err()); + let shared = MmField::Shared(MmSharedField { + batch_size: 1, + keep_on_cpu: false, + }); + assert!( + info.prepare_preprocessed(vec![feature(0, batched()), feature(2, shared)], 4) + .is_err() + ); + } +} diff --git a/rust/src/server/Cargo.toml b/rust/src/server/Cargo.toml index e4d12bc96b..b9abe19c63 100644 --- a/rust/src/server/Cargo.toml +++ b/rust/src/server/Cargo.toml @@ -20,6 +20,7 @@ libc.workspace = true llm-multimodal.workspace = true openssl.workspace = true prost-types.workspace = true +rmp-serde.workspace = true rmpv.workspace = true serde.workspace = true serde_json.workspace = true @@ -58,7 +59,6 @@ async-openai = { workspace = true, features = ["full"] } bytes.workspace = true clap.workspace = true expect-test.workspace = true -rmp-serde.workspace = true serial_test.workspace = true tempfile.workspace = true tokio = { workspace = true, features = ["test-util"] } diff --git a/rust/src/server/src/grpc/convert.rs b/rust/src/server/src/grpc/convert.rs index ca44f59c93..cf0045f714 100644 --- a/rust/src/server/src/grpc/convert.rs +++ b/rust/src/server/src/grpc/convert.rs @@ -5,9 +5,7 @@ //! request/response types. use tonic::Status; -use url::Url; use uuid::Uuid; -use vllm_chat::MediaContentPart; use vllm_engine_core_client::protocol::output::StopReason; use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsParams; use vllm_text::{ @@ -17,95 +15,6 @@ use vllm_text::{ use super::pb; -pub fn media_parts_from_request( - media: Vec, -) -> Result, Status> { - let mut parts = Vec::with_capacity(media.len()); - for (index, item) in media.into_iter().enumerate() { - let modality = item.modality(); - if modality == pb::Modality::Unspecified { - return Err(Status::invalid_argument(format!( - "media[{index}].modality is required" - ))); - } - let uuid = (!item.uuid.is_empty()).then_some(item.uuid); - let mime_type = (!item.mime_type.is_empty()).then_some(item.mime_type); - let source = item.source.ok_or_else(|| { - Status::invalid_argument(format!("media[{index}].source is required")) - })?; - match &source { - pb::media_item::Source::Url(url) => { - validate_media_uri(index, "url", url, &["http", "https"])?; - } - pb::media_item::Source::DataUri(uri) => { - validate_media_uri(index, "data_uri", uri, &["data"])?; - } - pb::media_item::Source::RawBytes(_) => {} - } - let part = match (modality, source) { - ( - pb::Modality::Image, - pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url), - ) => MediaContentPart::ImageUrl { - url, - detail: None, - uuid, - }, - (pb::Modality::Image, pb::media_item::Source::RawBytes(data)) => { - MediaContentPart::ImageData { - data, - mime_type, - uuid, - detail: None, - } - } - ( - pb::Modality::Video, - pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url), - ) => MediaContentPart::VideoUrl { url, uuid }, - (pb::Modality::Video, pb::media_item::Source::RawBytes(data)) => { - MediaContentPart::VideoData { - data, - mime_type, - uuid, - } - } - ( - pb::Modality::Audio, - pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url), - ) => MediaContentPart::AudioUrl { url, uuid }, - (pb::Modality::Audio, pb::media_item::Source::RawBytes(data)) => { - MediaContentPart::AudioData { - data, - mime_type, - uuid, - } - } - (pb::Modality::Unspecified, _) => unreachable!("modality validated above"), - }; - parts.push(part); - } - Ok(parts) -} - -fn validate_media_uri( - index: usize, - field: &str, - value: &str, - allowed_schemes: &[&str], -) -> Result<(), Status> { - let uri = Url::parse(value).map_err(|_| { - Status::invalid_argument(format!("media[{index}].{field} is not a valid URI")) - })?; - if !allowed_schemes.contains(&uri.scheme()) { - return Err(Status::invalid_argument(format!( - "media[{index}].{field} must use the {} scheme", - allowed_schemes.join(" or ") - ))); - } - Ok(()) -} - // ======================================================================================== // Request conversion // ======================================================================================== diff --git a/rust/src/server/src/grpc/inference.rs b/rust/src/server/src/grpc/inference.rs index 7051558cd8..72943b6af0 100644 --- a/rust/src/server/src/grpc/inference.rs +++ b/rust/src/server/src/grpc/inference.rs @@ -149,19 +149,23 @@ impl InferenceServiceImpl { })?); } - let media = convert::media_parts_from_request(media)?; + let media = super::media::from_proto(media)?; if !media.is_empty() { let Prompt::TokenIds(mut token_ids) = text_request.prompt else { return Err(Status::invalid_argument( "multimodal gRPC requests must provide token_ids input", )); }; - let mm_features = self - .state - .chat - .prepare_media(media, &mut token_ids) - .await - .map_err(|error| Status::internal(error.to_report_string()))?; + let mm_features = + self.state.chat.prepare_media(media, &mut token_ids).await.map_err( + |error| { + if error.is_request_validation_error() { + Status::invalid_argument(error.to_report_string()) + } else { + Status::internal(error.to_report_string()) + } + }, + )?; text_request.prompt = Prompt::TokenIds(token_ids); text_request.mm_features = mm_features; } diff --git a/rust/src/server/src/grpc/media.rs b/rust/src/server/src/grpc/media.rs new file mode 100644 index 0000000000..ff746b191e --- /dev/null +++ b/rust/src/server/src/grpc/media.rs @@ -0,0 +1,341 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! Protobuf media adaptation into the shared multimodal preparation boundary. + +use std::io::Cursor; + +use serde::Deserialize as _; +use thiserror_ext::AsReport as _; +use tonic::Status; +use url::Url; +use vllm_chat::MediaContentPart; +use vllm_chat::multimodal::MultimodalInput; +use vllm_engine_core_client::protocol::multimodal::{ + MmFeatureSpec, MmKwargsItem, MmModality, PlaceholderRange, +}; +use vllm_engine_core_client::protocol::tensor::WireTensor; + +use super::pb; + +/// Maximum total serialized kwargs size accepted in one gRPC request. +const MAX_INLINE_MM_BYTES: usize = 16 * 1024 * 1024; +const MAX_MSGPACK_DEPTH: usize = 32; + +/// Adapt a request containing either raw media or preprocessed features. +/// +/// Require a single input kind and bound the total kwargs size before decoding. +/// Shared media preparation validates features, model support, item limits, and prompt bounds. +pub(super) fn from_proto(media: Vec) -> Result { + if !media + .iter() + .any(|item| matches!(item.source, Some(pb::media_item::Source::Features(_)))) + { + return raw_parts_from_proto(media).map(MultimodalInput::Raw); + } + let mut encoded_bytes = 0usize; + for item in &media { + let Some(pb::media_item::Source::Features(feature)) = &item.source else { + return Err(mixed_media_error()); + }; + encoded_bytes = encoded_bytes.saturating_add(feature.kwargs.as_ref().map_or(0, Vec::len)); + } + if encoded_bytes > MAX_INLINE_MM_BYTES { + return Err(Status::resource_exhausted(format!( + "inline multimodal payload exceeds {MAX_INLINE_MM_BYTES} bytes" + ))); + } + let features = media + .into_iter() + .enumerate() + .map(|(index, item)| { + feature_from_proto(item).map_err(|error| { + Status::new(error.code(), format!("media[{index}]: {}", error.message())) + }) + }) + .collect::, _>>()?; + Ok(MultimodalInput::Preprocessed(features)) +} + +fn feature_from_proto(item: pb::MediaItem) -> Result { + let modality = match item.modality() { + pb::Modality::Image => MmModality::Image, + pb::Modality::Video => MmModality::Video, + pb::Modality::Audio => MmModality::Audio, + pb::Modality::Unspecified => return Err(Status::invalid_argument("modality is required")), + }; + let Some(pb::media_item::Source::Features(feature)) = item.source else { + return Err(mixed_media_error()); + }; + let bytes = feature.kwargs.ok_or_else(|| { + Status::invalid_argument("features.kwargs is required; cache-only features are unsupported") + })?; + let offset = usize::try_from(feature.offset) + .map_err(|_| Status::invalid_argument("features.offset exceeds platform limits"))?; + let length = usize::try_from(feature.length) + .map_err(|_| Status::invalid_argument("features.length exceeds platform limits"))?; + let is_embed = if feature.is_embed.is_empty() { + None + } else { + Some( + WireTensor::from_bool(vec![length], feature.is_embed) + .map_err(Status::invalid_argument)?, + ) + }; + Ok(MmFeatureSpec { + modality, + data: Some(decode_inline_mm_kwargs(&bytes)?), + identifier: feature.identifier, + mm_hash: feature.mm_hash, + mm_position: PlaceholderRange { + offset, + length, + is_embed, + }, + }) +} + +/// Decode one inline item using the engine protocol's typed serde schema. +fn decode_inline_mm_kwargs(bytes: &[u8]) -> Result { + let mut decoder = rmp_serde::Deserializer::new(Cursor::new(bytes)); + decoder.set_max_depth(MAX_MSGPACK_DEPTH); + let kwargs = MmKwargsItem::deserialize(&mut decoder).map_err(|error| { + Status::invalid_argument(format!( + "invalid multimodal kwargs MessagePack: {}", + error.as_report() + )) + })?; + if decoder.position() != bytes.len() as u64 { + return Err(Status::invalid_argument( + "kwargs contains trailing MessagePack data", + )); + } + Ok(kwargs) +} + +fn mixed_media_error() -> Status { + Status::invalid_argument("raw media and preprocessed media features cannot be mixed") +} + +fn raw_parts_from_proto(media: Vec) -> Result, Status> { + let mut parts = Vec::with_capacity(media.len()); + for (index, item) in media.into_iter().enumerate() { + let modality = item.modality(); + if modality == pb::Modality::Unspecified { + return Err(Status::invalid_argument(format!( + "media[{index}].modality is required" + ))); + } + let uuid = (!item.uuid.is_empty()).then_some(item.uuid); + let mime_type = (!item.mime_type.is_empty()).then_some(item.mime_type); + let source = item.source.ok_or_else(|| { + Status::invalid_argument(format!("media[{index}].source is required")) + })?; + match &source { + pb::media_item::Source::Url(url) => { + validate_media_uri(index, "url", url, &["http", "https"])?; + } + pb::media_item::Source::DataUri(uri) => { + validate_media_uri(index, "data_uri", uri, &["data"])?; + } + pb::media_item::Source::RawBytes(_) => {} + pb::media_item::Source::Features(_) => return Err(mixed_media_error()), + } + let part = match (modality, source) { + ( + pb::Modality::Image, + pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url), + ) => MediaContentPart::ImageUrl { + url, + detail: None, + uuid, + }, + (pb::Modality::Image, pb::media_item::Source::RawBytes(data)) => { + MediaContentPart::ImageData { + data, + mime_type, + uuid, + detail: None, + } + } + ( + pb::Modality::Video, + pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url), + ) => MediaContentPart::VideoUrl { url, uuid }, + (pb::Modality::Video, pb::media_item::Source::RawBytes(data)) => { + MediaContentPart::VideoData { + data, + mime_type, + uuid, + } + } + ( + pb::Modality::Audio, + pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url), + ) => MediaContentPart::AudioUrl { url, uuid }, + (pb::Modality::Audio, pb::media_item::Source::RawBytes(data)) => { + MediaContentPart::AudioData { + data, + mime_type, + uuid, + } + } + (_, pb::media_item::Source::Features(_)) => return Err(mixed_media_error()), + (pb::Modality::Unspecified, _) => unreachable!("modality validated above"), + }; + parts.push(part); + } + Ok(parts) +} + +fn validate_media_uri( + index: usize, + field: &str, + value: &str, + allowed_schemes: &[&str], +) -> Result<(), Status> { + let uri = Url::parse(value).map_err(|_| { + Status::invalid_argument(format!("media[{index}].{field} is not a valid URI")) + })?; + if !allowed_schemes.contains(&uri.scheme()) { + return Err(Status::invalid_argument(format!( + "media[{index}].{field} must use the {} scheme", + allowed_schemes.join(" or ") + ))); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use super::*; + use vllm_engine_core_client::protocol::multimodal::{ + MmBatchedField, MmField, MmFieldElem, MmFlatField, MmKwargValue, MmSlice, SliceSpec, + }; + + fn preprocessed() -> pb::MediaItem { + let kwargs = BTreeMap::from([( + "x", + MmFieldElem { + data: Some(MmKwargValue::Int(1)), + field: MmField::Batched(MmBatchedField { keep_on_cpu: false }), + }, + )]); + pb::MediaItem { + modality: pb::Modality::Image as i32, + source: Some(pb::media_item::Source::Features( + pb::PreprocessedMediaFeatures { + kwargs: Some(rmp_serde::to_vec_named(&kwargs).unwrap()), + identifier: "image-id".to_owned(), + length: 1, + ..Default::default() + }, + )), + ..Default::default() + } + } + + #[test] + fn mixed_media_is_rejected_in_both_orders() { + let raw = pb::MediaItem { + modality: pb::Modality::Image as i32, + source: Some(pb::media_item::Source::RawBytes(vec![1])), + ..Default::default() + }; + for media in [vec![raw.clone(), preprocessed()], vec![preprocessed(), raw]] { + assert_eq!( + from_proto(media).unwrap_err().code(), + tonic::Code::InvalidArgument + ); + } + } + + #[test] + fn protobuf_features_require_modality_inline_kwargs_and_a_matching_mask() { + let mut missing_modality = preprocessed(); + missing_modality.modality = 99; + let mut missing_kwargs = preprocessed(); + let Some(pb::media_item::Source::Features(feature)) = &mut missing_kwargs.source else { + unreachable!() + }; + feature.kwargs = None; + let mut invalid_mask = preprocessed(); + let Some(pb::media_item::Source::Features(feature)) = &mut invalid_mask.source else { + unreachable!() + }; + feature.is_embed = vec![true, false]; + for media in [missing_modality, missing_kwargs, invalid_mask] { + let error = from_proto(vec![media]).unwrap_err(); + assert_eq!(error.code(), tonic::Code::InvalidArgument); + assert!(error.message().starts_with("media[0]:")); + } + } + + #[test] + fn aggregate_payload_limit_is_checked_before_decoding_items() { + let mut item = preprocessed(); + let Some(pb::media_item::Source::Features(feature)) = &mut item.source else { + unreachable!() + }; + feature.kwargs = Some(vec![0; MAX_INLINE_MM_BYTES / 2 + 1]); + let error = from_proto(vec![item.clone(), item]).unwrap_err(); + assert_eq!(error.code(), tonic::Code::ResourceExhausted); + } + + #[test] + fn kwargs_decoder_rejects_trailing_truncated_and_deeply_nested_input() { + let kwargs = BTreeMap::from([( + "pixels".to_owned(), + MmFieldElem { + data: Some(MmKwargValue::Int(1)), + field: MmField::Batched(MmBatchedField { keep_on_cpu: false }), + }, + )]); + let valid = rmp_serde::to_vec_named(&kwargs).unwrap(); + assert_eq!(decode_inline_mm_kwargs(&valid).unwrap(), kwargs); + let mut trailing = valid.clone(); + trailing.push(0); + let mut nested_kwargs = kwargs.clone(); + let mut value = MmKwargValue::Int(1); + for _ in 0..64 { + value = MmKwargValue::List(vec![value]); + } + nested_kwargs.get_mut("pixels").unwrap().data = Some(value); + let nested = rmp_serde::to_vec_named(&nested_kwargs).unwrap(); + // A declared array length must not drive a proportional allocation. + let huge_array = vec![ + 0x81, 0xa1, b'x', 0x81, 0xa4, b'd', b'a', b't', b'a', 0xdd, 0xff, 0xff, 0xff, 0xff, + ]; + for bytes in [&valid[..valid.len() - 1], &trailing, &nested, &huge_array] { + assert!(decode_inline_mm_kwargs(bytes).is_err()); + } + } + + #[test] + fn kwargs_decode_python_encoder_tensor_and_factory_tuple() { + // MsgpackEncoder(size_threshold=1 << 30), vLLM 6cbb3c154e: + // pixel_values = tensor([1., 2.]), MultiModalFlatField(slices=[slice(0, 2)]). + let bytes = b"\x81\xacpixel_values\x82\xa4data\x93\xa7float32\x91\x02\xd7\x03\ + \x00\x00\x80\x3f\x00\x00\x00\x40\ + \xa5field\x92\xa4flat\x83\xabkeep_on_cpu\xc2\ + \xa6slices\x91\x93\x00\x02\xc0\xa3dim\x00"; + let kwargs = decode_inline_mm_kwargs(bytes).unwrap(); + let expected = MmFieldElem { + data: Some(MmKwargValue::Tensor( + WireTensor::from_f32(vec![2], vec![1., 2.]).unwrap(), + )), + field: MmField::Flat(MmFlatField { + slices: vec![MmSlice::Slice(SliceSpec { + start: Some(0), + stop: Some(2), + step: None, + })], + dim: 0, + keep_on_cpu: false, + }), + }; + assert_eq!(kwargs["pixel_values"], expected); + } +} diff --git a/rust/src/server/src/grpc/mod.rs b/rust/src/server/src/grpc/mod.rs index 3c96afd9b8..c61c1e5e14 100644 --- a/rust/src/server/src/grpc/mod.rs +++ b/rust/src/server/src/grpc/mod.rs @@ -7,6 +7,7 @@ mod control; mod convert; mod health; mod inference; +mod media; /// Generated protobuf/gRPC types for the `vllm` package. pub use vllm_proto as pb; diff --git a/rust/src/server/src/grpc/tests.rs b/rust/src/server/src/grpc/tests.rs index 59403277ee..f13dc11b27 100644 --- a/rust/src/server/src/grpc/tests.rs +++ b/rust/src/server/src/grpc/tests.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project +use std::collections::BTreeMap; use std::fs; use std::future::Future; use std::io; @@ -23,6 +24,7 @@ use tonic_health::pb::health_check_response::ServingStatus as HealthServingStatu use tonic_health::pb::health_client::HealthClient; use tonic_health::server::health_reporter; use tower::service_fn; +use vllm_chat::multimodal::{MmLimitPerPrompt, MmLimitSpec, MmModality}; use vllm_chat::{ ChatBackend, ChatLlm, ChatRenderer, ChatRequest, ChatTextBackend, DefaultChatOutputProcessor, DynChatOutputProcessor, DynChatRenderer, NewChatOutputProcessorOptions, RenderedPrompt, @@ -32,12 +34,17 @@ use vllm_engine_core_client::mock_engine::{ default_ready_response, }; use vllm_engine_core_client::protocol::decode_value; +use vllm_engine_core_client::protocol::dtype::TensorDtype; use vllm_engine_core_client::protocol::handshake::{EngineCoreReadyResponse, KvEventsConfig}; +use vllm_engine_core_client::protocol::multimodal::{ + MmBatchedField, MmField, MmFieldElem, MmKwargValue, +}; use vllm_engine_core_client::protocol::output::{ EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, RequestBatchOutputs, UtilityCallOutput, }; use vllm_engine_core_client::protocol::request::EngineCoreRequest; +use vllm_engine_core_client::protocol::tensor::WireTensor; use vllm_engine_core_client::protocol::utility::{UtilityOutput, UtilityResultEnvelope}; use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task_with_ready}; use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId, TransportMode}; @@ -292,6 +299,12 @@ const QWEN_IMAGE_TOKEN_ID: u32 = 151655; const TINY_PNG_DATA_URI: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="; fn multimodal_backend() -> Arc { + multimodal_backend_with_limits(Default::default()) +} + +fn multimodal_backend_with_limits( + limit_mm_per_prompt: MmLimitPerPrompt, +) -> Arc { let config_path = std::env::temp_dir().join(format!( "vllm-grpc-qwen-config-{}.json", uuid::Uuid::new_v4() @@ -309,7 +322,7 @@ fn multimodal_backend() -> Arc { ..Default::default() }, Arc::new(TestTokenizer::new().with_regular_token("<|image_pad|>", QWEN_IMAGE_TOKEN_ID)), - Default::default(), + limit_mm_per_prompt, ) .expect("load multimodal info") .expect("qwen multimodal info is registered"); @@ -877,6 +890,249 @@ async fn unary_generate_prepares_multimodal_input_for_engine_core() { server_task.abort(); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn generate_forwards_preprocessed_multimodal_features() { + for stream in [false, true] { + let kwargs = BTreeMap::from([( + "pixel_values".to_string(), + MmFieldElem { + data: Some(MmKwargValue::Tensor(WireTensor::from_raw( + TensorDtype::U8, + vec![3], + vec![1, 2, 3], + ))), + field: MmField::Batched(MmBatchedField { keep_on_cpu: false }), + }, + )]); + let encoded_kwargs = rmp_serde::to_vec_named(&kwargs).expect("encode multimodal kwargs"); + let expected_kwargs = kwargs.clone(); + let (inference_service, control_service, engine_health, engine_task) = + setup_grpc_service_with_backend( + b"engine-grpc-preprocessed-multimodal", + default_stream_output_specs(), + multimodal_backend(), + move |request| { + assert_eq!( + request.prompt_token_ids.as_deref(), + Some(&[11, 99, 99, 12][..]) + ); + let features = request.mm_features.as_ref().expect("multimodal features"); + assert_eq!(features.len(), 1); + let feature = &features[0]; + assert_eq!(feature.modality.as_str(), "image"); + assert_eq!(feature.identifier, "image-hash-a"); + assert_eq!(feature.mm_hash.as_deref(), Some("image-hash-a")); + assert_eq!(feature.mm_position.offset, 1); + assert_eq!(feature.mm_position.length, 2); + assert_eq!(feature.data.as_ref(), Some(&expected_kwargs)); + assert_eq!( + feature.mm_position.is_embed, + Some(WireTensor::from_bool(vec![2], vec![true, false]).unwrap()) + ); + }, + ) + .await; + let (channel, server_task) = start_grpc_test_server( + inference_service, + control_service, + engine_health, + tokio_util::sync::CancellationToken::new(), + ) + .await; + let mut client = InferenceClient::new(channel); + + let request = pb::GenerateRequest { + request_id: "test-preprocessed-multimodal".to_string(), + model: "test-model".to_string(), + prompt: Some(pb::generate_request::Prompt::TokenIds(pb::TokenIds { + ids: vec![11, 99, 99, 12], + })), + media: vec![pb::MediaItem { + modality: pb::Modality::Image as i32, + source: Some(pb::media_item::Source::Features( + pb::PreprocessedMediaFeatures { + kwargs: Some(encoded_kwargs), + identifier: "image-hash-a".to_string(), + offset: 1, + length: 2, + mm_hash: Some("image-hash-a".to_string()), + is_embed: vec![true, false], + }, + )), + mime_type: String::new(), + uuid: String::new(), + }], + stopping: Some(pb::StoppingCriteria { + max_new_tokens: 10, + ..Default::default() + }), + ..Default::default() + }; + if stream { + let mut response = + client.generate_stream(request).await.expect("preprocessed stream").into_inner(); + while response.message().await.expect("stream message").is_some() {} + } else { + client.generate(request).await.expect("preprocessed generate"); + } + + engine_task.await.expect("mock engine task"); + server_task.abort(); + } +} + +fn preprocessed_grpc_media( + modality: pb::Modality, + identifier: &str, + offset: u64, + length: u64, +) -> pb::MediaItem { + let kwargs = BTreeMap::from([( + "pixel_values".to_string(), + MmFieldElem { + data: Some(MmKwargValue::Tensor(WireTensor::from_raw( + TensorDtype::U8, + vec![1], + vec![1], + ))), + field: MmField::Batched(MmBatchedField { keep_on_cpu: false }), + }, + )]); + pb::MediaItem { + modality: modality as i32, + source: Some(pb::media_item::Source::Features( + pb::PreprocessedMediaFeatures { + kwargs: Some(rmp_serde::to_vec_named(&kwargs).expect("encode kwargs")), + identifier: identifier.to_string(), + offset, + length, + mm_hash: Some(identifier.to_string()), + is_embed: Vec::new(), + }, + )), + mime_type: String::new(), + uuid: String::new(), + } +} + +fn preprocessed_grpc_request(media: Vec) -> pb::GenerateRequest { + pb::GenerateRequest { + request_id: "test-preprocessed-validation".to_string(), + model: "test-model".to_string(), + prompt: Some(pb::generate_request::Prompt::TokenIds(pb::TokenIds { + ids: vec![11, 99, 99, 12], + })), + media, + stopping: Some(pb::StoppingCriteria { + max_new_tokens: 10, + ..Default::default() + }), + ..Default::default() + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn unary_generate_rejects_preprocessed_features_for_text_only_model() { + let (inference_service, control_service, engine_health, _engine_task) = + setup_grpc_service_with_backend( + b"engine-grpc-preprocessed-text-only", + default_stream_output_specs(), + Arc::new(FakeTextBackend), + |_| {}, + ) + .await; + let (channel, server_task) = start_grpc_test_server( + inference_service, + control_service, + engine_health, + tokio_util::sync::CancellationToken::new(), + ) + .await; + let mut client = InferenceClient::new(channel); + + let status = client + .generate(preprocessed_grpc_request(vec![preprocessed_grpc_media( + pb::Modality::Image, + "image-a", + 1, + 2, + )])) + .await + .expect_err("text-only models must reject preprocessed features"); + + assert_eq!(status.code(), tonic::Code::InvalidArgument); + server_task.abort(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn generate_validates_preprocessed_model_limits_and_prompt() { + let limits = MmLimitPerPrompt::from([(MmModality::Image, MmLimitSpec::Count(1))]); + let (inference_service, control_service, engine_health, _engine_task) = + setup_grpc_service_with_backend( + b"engine-grpc-preprocessed-limit", + default_stream_output_specs(), + multimodal_backend_with_limits(limits), + |_| {}, + ) + .await; + let (channel, server_task) = start_grpc_test_server( + inference_service, + control_service, + engine_health, + tokio_util::sync::CancellationToken::new(), + ) + .await; + let mut client = InferenceClient::new(channel); + + let over_limit = preprocessed_grpc_request(vec![ + preprocessed_grpc_media(pb::Modality::Image, "image-a", 0, 1), + preprocessed_grpc_media(pb::Modality::Image, "image-b", 2, 1), + ]); + let bad_range = preprocessed_grpc_request(vec![preprocessed_grpc_media( + pb::Modality::Image, + "image", + 3, + 2, + )]); + let unsupported = preprocessed_grpc_request(vec![preprocessed_grpc_media( + pb::Modality::Audio, + "audio", + 0, + 1, + )]); + let mut text = preprocessed_grpc_request(vec![preprocessed_grpc_media( + pb::Modality::Image, + "image", + 0, + 1, + )]); + let mut truncated = text.clone(); + text.prompt = Some(pb::generate_request::Prompt::Text("hello".to_owned())); + truncated.truncate_prompt_tokens = 1; + let empty_identifier = + preprocessed_grpc_request(vec![preprocessed_grpc_media(pb::Modality::Image, "", 0, 1)]); + for request in [ + over_limit, + bad_range, + unsupported, + text, + truncated, + empty_identifier, + ] { + let status = client.generate(request.clone()).await.expect_err("invalid multimodal input"); + assert_eq!(status.code(), tonic::Code::InvalidArgument); + let status = client + .generate_stream(request) + .await + .expect_err("invalid streaming multimodal input"); + assert_eq!(status.code(), tonic::Code::InvalidArgument); + } + server_task.abort(); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn streaming_generate_rejects_text_prompt_with_media() {