[Rust Frontend][Multimodal] Accept preprocessed multimodal gRPC features (#55047)

Signed-off-by: Biswa Panda <biswa.panda@gmail.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Biswa Panda
2026-09-12 02:29:49 +00:00
committed by GitHub
co-authored by Bugen Zhao Codex
parent a0844fa6c6
commit 31759ccb6d
16 changed files with 986 additions and 113 deletions
+1 -1
View File
@@ -5940,7 +5940,7 @@ dependencies = [
[[package]]
name = "vllm-proto"
version = "0.1.0"
version = "0.2.0"
dependencies = [
"prost",
"prost-types",
+1 -1
View File
@@ -147,7 +147,7 @@ vllm-llm = { path = "src/llm" }
vllm-managed-engine = { path = "src/managed-engine" }
vllm-metrics = { path = "src/metrics" }
vllm-parser = { path = "src/parser" }
vllm-proto = { version = "0.1.0", path = "proto" }
vllm-proto = { version = "0.2.0", path = "proto" }
vllm-server = { path = "src/server" }
vllm-text = { path = "src/text" }
vllm-tokenizer = { path = "src/tokenizer" }
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "vllm-proto"
version = "0.1.0"
version = "0.2.0"
edition.workspace = true
license.workspace = true
description = "Protobuf messages and gRPC bindings for the vLLM Rust frontend"
+1 -1
View File
@@ -17,7 +17,7 @@ Repository setup requires a `BUF_TOKEN` GitHub Actions secret with permission to
```toml
[dependencies]
vllm-proto = "0.1"
vllm-proto = "0.2"
```
For example, import `vllm_proto::inference_client::InferenceClient` or
+13
View File
@@ -228,7 +228,20 @@ message MediaItem {
string url = 2; // http:// or https://
string data_uri = 3; // data:
bytes raw_bytes = 4;
PreprocessedMediaFeatures features = 7;
}
string mime_type = 5;
string uuid = 6;
}
// Engine-ready multimodal features produced by a version-compatible frontend.
// `kwargs` is the MessagePack wire representation of one MmKwargsItem.
message PreprocessedMediaFeatures {
// Required inline payload. Cache-only references are not accepted.
optional bytes kwargs = 1;
string identifier = 2;
uint64 offset = 3;
uint64 length = 4;
optional string mm_hash = 5;
repeated bool is_embed = 6;
}
+3
View File
@@ -27,6 +27,8 @@ pub enum Error {
UnsupportedModality { modality: String },
#[error("At most {limit} {modality}(s) may be provided in one prompt.")]
MmLimitExceeded { modality: String, limit: usize },
#[error("invalid inline multimodal features: {message}")]
InvalidPreprocessedMultimodal { message: String },
#[error("multimodal preprocessing error: {0}")]
Multimodal(#[message] String),
#[error("{kind} parsing is not available for model `{model_id}`")]
@@ -106,6 +108,7 @@ impl Error {
Self::UnsupportedMultimodalRenderer
| Self::UnsupportedMultimodalContent(_)
| Self::UnsupportedModality { .. }
| Self::InvalidPreprocessedMultimodal { .. }
| Self::MmLimitExceeded { .. } => true,
_ => false,
+14 -5
View File
@@ -130,7 +130,7 @@ impl ChatRequestProcessor {
/// Prepare media for an already-tokenized request.
async fn prepare_media(
&self,
media: Vec<MediaContentPart>,
media: multimodal::MultimodalInput,
token_ids: &mut Vec<u32>,
) -> Result<Option<MmFeatures>> {
if media.is_empty() {
@@ -140,8 +140,15 @@ impl ChatRequestProcessor {
.backend
.multimodal_model_info()
.ok_or(Error::UnsupportedMultimodalRenderer)?;
let model_dtype = self.model_dtype.ok_or(Error::UnsupportedMultimodalRenderer)?;
let features = info.prepare_multimodal(media, token_ids, model_dtype).await?;
let features = match media {
multimodal::MultimodalInput::Raw(parts) => {
let model_dtype = self.model_dtype.ok_or(Error::UnsupportedMultimodalRenderer)?;
info.prepare_multimodal(parts, token_ids, model_dtype).await?
}
multimodal::MultimodalInput::Preprocessed(features) => {
info.prepare_preprocessed(features, token_ids.len())?
}
};
Ok(Some(features))
}
@@ -286,12 +293,14 @@ impl ChatLlm {
}
/// Prepare media for an already-tokenized request.
/// Raw content is preprocessed; inline features are checked against model
/// capabilities, modality limits, and the final prompt positions.
pub async fn prepare_media(
&self,
media: Vec<MediaContentPart>,
media: impl Into<multimodal::MultimodalInput>,
token_ids: &mut Vec<u32>,
) -> Result<Option<MmFeatures>> {
self.processor.prepare_media(media, token_ids).await
self.processor.prepare_media(media.into(), token_ids).await
}
/// Effective tool-call parser name for this model, if parsing is enabled.
+38 -4
View File
@@ -39,11 +39,14 @@ use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest};
mod audio;
mod expand;
mod image;
mod input;
mod item;
mod preprocessed;
mod tensor;
mod video;
use self::expand::expand_prompt_token_ids;
pub use self::input::MultimodalInput;
/// Resolved multimodal support for one loaded model.
#[derive(Clone)]
@@ -654,11 +657,16 @@ impl MultimodalModelInfo {
///
/// Modalities without a configured count are unlimited.
fn validate_mm_limits(&self, media_parts: &[MediaContentPart]) -> Result<()> {
self.validate_modality_limits(media_parts.iter().filter_map(media_part_limit_modality))
}
fn validate_modality_limits(
&self,
modalities: impl IntoIterator<Item = MmModality>,
) -> Result<()> {
let mut counts: HashMap<MmModality, usize> = HashMap::new();
for part in media_parts {
if let Some(modality) = media_part_limit_modality(part) {
*counts.entry(modality).or_default() += 1;
}
for modality in modalities {
*counts.entry(modality).or_default() += 1;
}
for (modality, count) in counts {
@@ -677,6 +685,32 @@ impl MultimodalModelInfo {
Ok(())
}
/// Validate inline storage, batching, and placeholder ranges, then check
/// this model's supported modalities and item-count limits.
///
/// `prompt_len` must include all expanded multimodal placeholders.
pub(crate) fn prepare_preprocessed(
&self,
mut features: MmFeatures,
prompt_len: usize,
) -> Result<MmFeatures> {
preprocessed::validate_features(&mut features, prompt_len)?;
for feature in &features {
let supported = match feature.modality {
MmModality::Image => self.image.is_some(),
MmModality::Video => self.video.is_some(),
MmModality::Audio => self.audio.is_some(),
};
if !supported {
return Err(Error::UnsupportedModality {
modality: feature.modality.as_str().to_owned(),
});
}
}
self.validate_modality_limits(features.iter().map(|feature| feature.modality))?;
Ok(features)
}
/// Run media fetch, per-modality preprocessing, prompt expansion, and
/// feature build.
///
+29
View File
@@ -0,0 +1,29 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use llm_multimodal::MediaContentPart;
use vllm_engine_core_client::protocol::multimodal::MmFeatures;
/// Media preparation stage supplied by a frontend handler.
#[derive(Debug)]
pub enum MultimodalInput {
/// Raw media to fetch and preprocess, expanding prompt placeholders as needed.
Raw(Vec<MediaContentPart>),
/// Preprocessed features to validate against storage, batching, model, and prompt constraints.
Preprocessed(MmFeatures),
}
impl MultimodalInput {
pub fn is_empty(&self) -> bool {
match self {
Self::Raw(parts) => parts.is_empty(),
Self::Preprocessed(features) => features.is_empty(),
}
}
}
impl From<Vec<MediaContentPart>> for MultimodalInput {
fn from(parts: Vec<MediaContentPart>) -> Self {
Self::Raw(parts)
}
}
@@ -0,0 +1,274 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::collections::HashMap;
use vllm_engine_core_client::protocol::dtype::TensorDtype;
use vllm_engine_core_client::protocol::multimodal::{
MmFeatureSpec, MmField, MmKwargValue, MmKwargsItem,
};
use crate::error::{
Error, Result, bail_invalid_preprocessed_multimodal, invalid_preprocessed_multimodal,
};
const MAX_DEPTH: usize = 32;
/// Validate inline storage, placeholder ranges, and batching compatibility,
/// then stably sort features by their prompt offsets.
/// Encoder cache identifiers are supplied by the producer, which must bind
/// them to the feature content.
pub(super) fn validate_features(features: &mut [MmFeatureSpec], prompt_len: usize) -> Result<()> {
let mut fields = HashMap::new();
for feature in features.iter() {
if feature.identifier.is_empty() {
bail_invalid_preprocessed_multimodal!("feature identifier must be nonempty");
}
let data = feature.data.as_ref().ok_or_else(|| {
invalid_preprocessed_multimodal!(
"inline data is required; cache-only features are unsupported"
)
})?;
validate_kwargs(data)?;
for (key, elem) in data {
if let Some(previous) = fields.insert((feature.modality, key), &elem.field)
&& !same_batching(previous, &elem.field)
{
bail_invalid_preprocessed_multimodal!(
"incompatible batching for {} field {key:?}",
feature.modality.as_str()
);
}
}
let position = &feature.mm_position;
if position.length == 0 || position.offset.checked_add(position.length).is_none() {
bail_invalid_preprocessed_multimodal!(
"placeholder range must have a positive length and fit in usize"
);
}
if position.offset + position.length > prompt_len {
bail_invalid_preprocessed_multimodal!(
"{} placeholder range exceeds prompt token IDs",
feature.modality.as_str()
);
}
if let Some(mask) = &position.is_embed {
if mask.dtype != TensorDtype::Bool || mask.shape != [position.length] {
bail_invalid_preprocessed_multimodal!(
"is_embed must be a boolean tensor of shape [placeholder length]"
);
}
mask.validate_inline()
.map_err(|message| Error::InvalidPreprocessedMultimodal { message })?;
}
}
// Mirror the Python frontend (`argsort_mm_positions`): features are
// ordered by their placeholder position in the prompt.
features.sort_by_key(|feature| feature.mm_position.offset);
Ok(())
}
fn validate_kwargs(kwargs: &MmKwargsItem) -> Result<()> {
if kwargs.is_empty() {
bail_invalid_preprocessed_multimodal!("kwargs must contain at least one field");
}
for (key, elem) in kwargs {
if key.is_empty() {
bail_invalid_preprocessed_multimodal!("kwargs field name must be nonempty");
}
let data = elem.data.as_ref().ok_or_else(|| {
invalid_preprocessed_multimodal!("field {key:?} requires inline data")
})?;
validate_value(data, 0)?;
if let MmField::Flat(field) = &elem.field {
match data {
MmKwargValue::Tensor(tensor) => {
let rank = tensor.shape.len() as i32;
if rank == 0 || field.dim < -rank || field.dim >= rank {
bail_invalid_preprocessed_multimodal!(
"flat field {key:?} dim is outside the tensor rank"
);
}
}
MmKwargValue::List(_) if field.dim == 0 => {}
_ => bail_invalid_preprocessed_multimodal!(
"flat field {key:?} requires a tensor or a list with dim=0"
),
}
}
}
Ok(())
}
fn validate_value(value: &MmKwargValue, depth: usize) -> Result<()> {
if depth > MAX_DEPTH {
bail_invalid_preprocessed_multimodal!("kwargs nesting exceeds {MAX_DEPTH} levels");
}
match value {
MmKwargValue::Tensor(tensor) => tensor
.validate_inline()
.map_err(|message| Error::InvalidPreprocessedMultimodal { message }),
MmKwargValue::List(values) => {
for value in values {
validate_value(value, depth + 1)?;
}
Ok(())
}
MmKwargValue::Int(_) | MmKwargValue::Float(_) => Ok(()),
}
}
fn same_batching(left: &MmField, right: &MmField) -> bool {
// Slices and batch_size describe the producer's original batch. Python
// reduce_data combines the already-split item data without using them.
match (left, right) {
(MmField::Batched(a), MmField::Batched(b)) => a == b,
(MmField::Flat(a), MmField::Flat(b)) => a.dim == b.dim && a.keep_on_cpu == b.keep_on_cpu,
(MmField::Shared(a), MmField::Shared(b)) => a.keep_on_cpu == b.keep_on_cpu,
_ => false,
}
}
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
use vllm_engine_core_client::protocol::multimodal::{
MmBatchedField, MmFieldElem, MmFlatField, MmModality, MmSharedField, MmSlice,
PlaceholderRange, SliceSpec,
};
use vllm_engine_core_client::protocol::tensor::{WireArrayData, WireTensor};
use super::*;
use crate::multimodal::tests::qwen3_vl_info;
fn feature(offset: usize, field: MmField) -> MmFeatureSpec {
MmFeatureSpec {
data: Some(BTreeMap::from([(
"pixels".to_owned(),
MmFieldElem {
data: Some(MmKwargValue::Tensor(
WireTensor::from_f32(vec![2], vec![1., 2.]).unwrap(),
)),
field,
},
)])),
modality: MmModality::Image,
identifier: format!("image-{offset}"),
mm_position: PlaceholderRange {
offset,
length: 2,
is_embed: None,
},
mm_hash: Some("processor-hash".to_owned()),
}
}
fn batched() -> MmField {
MmField::Batched(MmBatchedField { keep_on_cpu: false })
}
#[test]
fn inline_features_preserve_overlaps_and_stably_sort_prompt_positions() {
let info = qwen3_vl_info();
let mut a = feature(2, batched());
a.mm_position.is_embed = Some(WireTensor::from_bool(vec![2], vec![true, false]).unwrap());
let b = feature(1, batched());
let mut c = feature(2, batched());
c.modality = MmModality::Video;
let expected = vec![b.clone(), a.clone(), c.clone()];
let features = info.prepare_preprocessed(vec![a, b, c], 4).unwrap();
assert_eq!(features, expected);
}
#[test]
fn independently_preprocessed_items_keep_their_original_batch_metadata() {
let info = qwen3_vl_info();
let flat = |length| {
MmField::Flat(MmFlatField {
slices: vec![MmSlice::Slice(SliceSpec {
start: Some(0),
stop: Some(length),
step: None,
})],
dim: 0,
keep_on_cpu: false,
})
};
let shared = |batch_size| {
MmField::Shared(MmSharedField {
batch_size,
keep_on_cpu: true,
})
};
for (a, b) in [(flat(2), flat(7)), (shared(1), shared(3))] {
let features = vec![feature(0, a), feature(2, b)];
let result = info.prepare_preprocessed(features.clone(), 4).unwrap();
assert_eq!(result, features);
}
}
#[test]
fn inline_features_reject_cache_references_and_invalid_tensor_storage() {
let info = qwen3_vl_info();
let original = feature(0, batched());
let mut cached = original.clone();
cached.data = None;
assert!(info.prepare_preprocessed(vec![cached], 4).is_err());
let mut field_cached = original.clone();
field_cached.data.as_mut().unwrap().get_mut("pixels").unwrap().data = None;
assert!(info.prepare_preprocessed(vec![field_cached], 4).is_err());
for tensor in [
WireTensor {
dtype: TensorDtype::F32,
shape: vec![2],
data: WireArrayData::AuxIndex(1),
},
WireTensor::from_raw(TensorDtype::F32, vec![2], vec![0; 4]),
WireTensor::from_raw(TensorDtype::F32, vec![usize::MAX, 2], vec![]),
] {
let mut item = original.clone();
item.data.as_mut().unwrap().get_mut("pixels").unwrap().data =
Some(MmKwargValue::Tensor(tensor));
assert!(info.prepare_preprocessed(vec![item], 4).is_err());
}
}
#[test]
fn inline_features_validate_placeholder_and_mask_bounds() {
let info = qwen3_vl_info();
for (offset, length) in [(usize::MAX, 2), (0, 0)] {
let mut item = feature(offset, batched());
item.mm_position.length = length;
assert!(info.prepare_preprocessed(vec![item], 4).is_err());
}
assert!(info.prepare_preprocessed(vec![feature(2, batched())], 3).is_err());
for mask in [
WireTensor::from_bool(vec![1], vec![true]).unwrap(),
WireTensor::from_u32(vec![2], vec![1, 0]).unwrap(),
] {
let mut item = feature(0, batched());
item.mm_position.is_embed = Some(mask);
assert!(info.prepare_preprocessed(vec![item], 4).is_err());
}
}
#[test]
fn inline_features_reject_incompatible_batching_and_flat_axes() {
let info = qwen3_vl_info();
let flat = MmField::Flat(MmFlatField {
slices: vec![],
dim: 2,
keep_on_cpu: false,
});
assert!(info.prepare_preprocessed(vec![feature(0, flat)], 4).is_err());
let shared = MmField::Shared(MmSharedField {
batch_size: 1,
keep_on_cpu: false,
});
assert!(
info.prepare_preprocessed(vec![feature(0, batched()), feature(2, shared)], 4)
.is_err()
);
}
}
+1 -1
View File
@@ -20,6 +20,7 @@ libc.workspace = true
llm-multimodal.workspace = true
openssl.workspace = true
prost-types.workspace = true
rmp-serde.workspace = true
rmpv.workspace = true
serde.workspace = true
serde_json.workspace = true
@@ -58,7 +59,6 @@ async-openai = { workspace = true, features = ["full"] }
bytes.workspace = true
clap.workspace = true
expect-test.workspace = true
rmp-serde.workspace = true
serial_test.workspace = true
tempfile.workspace = true
tokio = { workspace = true, features = ["test-util"] }
-91
View File
@@ -5,9 +5,7 @@
//! request/response types.
use tonic::Status;
use url::Url;
use uuid::Uuid;
use vllm_chat::MediaContentPart;
use vllm_engine_core_client::protocol::output::StopReason;
use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsParams;
use vllm_text::{
@@ -17,95 +15,6 @@ use vllm_text::{
use super::pb;
pub fn media_parts_from_request(
media: Vec<pb::MediaItem>,
) -> Result<Vec<MediaContentPart>, Status> {
let mut parts = Vec::with_capacity(media.len());
for (index, item) in media.into_iter().enumerate() {
let modality = item.modality();
if modality == pb::Modality::Unspecified {
return Err(Status::invalid_argument(format!(
"media[{index}].modality is required"
)));
}
let uuid = (!item.uuid.is_empty()).then_some(item.uuid);
let mime_type = (!item.mime_type.is_empty()).then_some(item.mime_type);
let source = item.source.ok_or_else(|| {
Status::invalid_argument(format!("media[{index}].source is required"))
})?;
match &source {
pb::media_item::Source::Url(url) => {
validate_media_uri(index, "url", url, &["http", "https"])?;
}
pb::media_item::Source::DataUri(uri) => {
validate_media_uri(index, "data_uri", uri, &["data"])?;
}
pb::media_item::Source::RawBytes(_) => {}
}
let part = match (modality, source) {
(
pb::Modality::Image,
pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url),
) => MediaContentPart::ImageUrl {
url,
detail: None,
uuid,
},
(pb::Modality::Image, pb::media_item::Source::RawBytes(data)) => {
MediaContentPart::ImageData {
data,
mime_type,
uuid,
detail: None,
}
}
(
pb::Modality::Video,
pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url),
) => MediaContentPart::VideoUrl { url, uuid },
(pb::Modality::Video, pb::media_item::Source::RawBytes(data)) => {
MediaContentPart::VideoData {
data,
mime_type,
uuid,
}
}
(
pb::Modality::Audio,
pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url),
) => MediaContentPart::AudioUrl { url, uuid },
(pb::Modality::Audio, pb::media_item::Source::RawBytes(data)) => {
MediaContentPart::AudioData {
data,
mime_type,
uuid,
}
}
(pb::Modality::Unspecified, _) => unreachable!("modality validated above"),
};
parts.push(part);
}
Ok(parts)
}
fn validate_media_uri(
index: usize,
field: &str,
value: &str,
allowed_schemes: &[&str],
) -> Result<(), Status> {
let uri = Url::parse(value).map_err(|_| {
Status::invalid_argument(format!("media[{index}].{field} is not a valid URI"))
})?;
if !allowed_schemes.contains(&uri.scheme()) {
return Err(Status::invalid_argument(format!(
"media[{index}].{field} must use the {} scheme",
allowed_schemes.join(" or ")
)));
}
Ok(())
}
// ========================================================================================
// Request conversion
// ========================================================================================
+11 -7
View File
@@ -149,19 +149,23 @@ impl InferenceServiceImpl {
})?);
}
let media = convert::media_parts_from_request(media)?;
let media = super::media::from_proto(media)?;
if !media.is_empty() {
let Prompt::TokenIds(mut token_ids) = text_request.prompt else {
return Err(Status::invalid_argument(
"multimodal gRPC requests must provide token_ids input",
));
};
let mm_features = self
.state
.chat
.prepare_media(media, &mut token_ids)
.await
.map_err(|error| Status::internal(error.to_report_string()))?;
let mm_features =
self.state.chat.prepare_media(media, &mut token_ids).await.map_err(
|error| {
if error.is_request_validation_error() {
Status::invalid_argument(error.to_report_string())
} else {
Status::internal(error.to_report_string())
}
},
)?;
text_request.prompt = Prompt::TokenIds(token_ids);
text_request.mm_features = mm_features;
}
+341
View File
@@ -0,0 +1,341 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//! Protobuf media adaptation into the shared multimodal preparation boundary.
use std::io::Cursor;
use serde::Deserialize as _;
use thiserror_ext::AsReport as _;
use tonic::Status;
use url::Url;
use vllm_chat::MediaContentPart;
use vllm_chat::multimodal::MultimodalInput;
use vllm_engine_core_client::protocol::multimodal::{
MmFeatureSpec, MmKwargsItem, MmModality, PlaceholderRange,
};
use vllm_engine_core_client::protocol::tensor::WireTensor;
use super::pb;
/// Maximum total serialized kwargs size accepted in one gRPC request.
const MAX_INLINE_MM_BYTES: usize = 16 * 1024 * 1024;
const MAX_MSGPACK_DEPTH: usize = 32;
/// Adapt a request containing either raw media or preprocessed features.
///
/// Require a single input kind and bound the total kwargs size before decoding.
/// Shared media preparation validates features, model support, item limits, and prompt bounds.
pub(super) fn from_proto(media: Vec<pb::MediaItem>) -> Result<MultimodalInput, Status> {
if !media
.iter()
.any(|item| matches!(item.source, Some(pb::media_item::Source::Features(_))))
{
return raw_parts_from_proto(media).map(MultimodalInput::Raw);
}
let mut encoded_bytes = 0usize;
for item in &media {
let Some(pb::media_item::Source::Features(feature)) = &item.source else {
return Err(mixed_media_error());
};
encoded_bytes = encoded_bytes.saturating_add(feature.kwargs.as_ref().map_or(0, Vec::len));
}
if encoded_bytes > MAX_INLINE_MM_BYTES {
return Err(Status::resource_exhausted(format!(
"inline multimodal payload exceeds {MAX_INLINE_MM_BYTES} bytes"
)));
}
let features = media
.into_iter()
.enumerate()
.map(|(index, item)| {
feature_from_proto(item).map_err(|error| {
Status::new(error.code(), format!("media[{index}]: {}", error.message()))
})
})
.collect::<Result<Vec<_>, _>>()?;
Ok(MultimodalInput::Preprocessed(features))
}
fn feature_from_proto(item: pb::MediaItem) -> Result<MmFeatureSpec, Status> {
let modality = match item.modality() {
pb::Modality::Image => MmModality::Image,
pb::Modality::Video => MmModality::Video,
pb::Modality::Audio => MmModality::Audio,
pb::Modality::Unspecified => return Err(Status::invalid_argument("modality is required")),
};
let Some(pb::media_item::Source::Features(feature)) = item.source else {
return Err(mixed_media_error());
};
let bytes = feature.kwargs.ok_or_else(|| {
Status::invalid_argument("features.kwargs is required; cache-only features are unsupported")
})?;
let offset = usize::try_from(feature.offset)
.map_err(|_| Status::invalid_argument("features.offset exceeds platform limits"))?;
let length = usize::try_from(feature.length)
.map_err(|_| Status::invalid_argument("features.length exceeds platform limits"))?;
let is_embed = if feature.is_embed.is_empty() {
None
} else {
Some(
WireTensor::from_bool(vec![length], feature.is_embed)
.map_err(Status::invalid_argument)?,
)
};
Ok(MmFeatureSpec {
modality,
data: Some(decode_inline_mm_kwargs(&bytes)?),
identifier: feature.identifier,
mm_hash: feature.mm_hash,
mm_position: PlaceholderRange {
offset,
length,
is_embed,
},
})
}
/// Decode one inline item using the engine protocol's typed serde schema.
fn decode_inline_mm_kwargs(bytes: &[u8]) -> Result<MmKwargsItem, Status> {
let mut decoder = rmp_serde::Deserializer::new(Cursor::new(bytes));
decoder.set_max_depth(MAX_MSGPACK_DEPTH);
let kwargs = MmKwargsItem::deserialize(&mut decoder).map_err(|error| {
Status::invalid_argument(format!(
"invalid multimodal kwargs MessagePack: {}",
error.as_report()
))
})?;
if decoder.position() != bytes.len() as u64 {
return Err(Status::invalid_argument(
"kwargs contains trailing MessagePack data",
));
}
Ok(kwargs)
}
fn mixed_media_error() -> Status {
Status::invalid_argument("raw media and preprocessed media features cannot be mixed")
}
fn raw_parts_from_proto(media: Vec<pb::MediaItem>) -> Result<Vec<MediaContentPart>, Status> {
let mut parts = Vec::with_capacity(media.len());
for (index, item) in media.into_iter().enumerate() {
let modality = item.modality();
if modality == pb::Modality::Unspecified {
return Err(Status::invalid_argument(format!(
"media[{index}].modality is required"
)));
}
let uuid = (!item.uuid.is_empty()).then_some(item.uuid);
let mime_type = (!item.mime_type.is_empty()).then_some(item.mime_type);
let source = item.source.ok_or_else(|| {
Status::invalid_argument(format!("media[{index}].source is required"))
})?;
match &source {
pb::media_item::Source::Url(url) => {
validate_media_uri(index, "url", url, &["http", "https"])?;
}
pb::media_item::Source::DataUri(uri) => {
validate_media_uri(index, "data_uri", uri, &["data"])?;
}
pb::media_item::Source::RawBytes(_) => {}
pb::media_item::Source::Features(_) => return Err(mixed_media_error()),
}
let part = match (modality, source) {
(
pb::Modality::Image,
pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url),
) => MediaContentPart::ImageUrl {
url,
detail: None,
uuid,
},
(pb::Modality::Image, pb::media_item::Source::RawBytes(data)) => {
MediaContentPart::ImageData {
data,
mime_type,
uuid,
detail: None,
}
}
(
pb::Modality::Video,
pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url),
) => MediaContentPart::VideoUrl { url, uuid },
(pb::Modality::Video, pb::media_item::Source::RawBytes(data)) => {
MediaContentPart::VideoData {
data,
mime_type,
uuid,
}
}
(
pb::Modality::Audio,
pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url),
) => MediaContentPart::AudioUrl { url, uuid },
(pb::Modality::Audio, pb::media_item::Source::RawBytes(data)) => {
MediaContentPart::AudioData {
data,
mime_type,
uuid,
}
}
(_, pb::media_item::Source::Features(_)) => return Err(mixed_media_error()),
(pb::Modality::Unspecified, _) => unreachable!("modality validated above"),
};
parts.push(part);
}
Ok(parts)
}
fn validate_media_uri(
index: usize,
field: &str,
value: &str,
allowed_schemes: &[&str],
) -> Result<(), Status> {
let uri = Url::parse(value).map_err(|_| {
Status::invalid_argument(format!("media[{index}].{field} is not a valid URI"))
})?;
if !allowed_schemes.contains(&uri.scheme()) {
return Err(Status::invalid_argument(format!(
"media[{index}].{field} must use the {} scheme",
allowed_schemes.join(" or ")
)));
}
Ok(())
}
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
use super::*;
use vllm_engine_core_client::protocol::multimodal::{
MmBatchedField, MmField, MmFieldElem, MmFlatField, MmKwargValue, MmSlice, SliceSpec,
};
fn preprocessed() -> pb::MediaItem {
let kwargs = BTreeMap::from([(
"x",
MmFieldElem {
data: Some(MmKwargValue::Int(1)),
field: MmField::Batched(MmBatchedField { keep_on_cpu: false }),
},
)]);
pb::MediaItem {
modality: pb::Modality::Image as i32,
source: Some(pb::media_item::Source::Features(
pb::PreprocessedMediaFeatures {
kwargs: Some(rmp_serde::to_vec_named(&kwargs).unwrap()),
identifier: "image-id".to_owned(),
length: 1,
..Default::default()
},
)),
..Default::default()
}
}
#[test]
fn mixed_media_is_rejected_in_both_orders() {
let raw = pb::MediaItem {
modality: pb::Modality::Image as i32,
source: Some(pb::media_item::Source::RawBytes(vec![1])),
..Default::default()
};
for media in [vec![raw.clone(), preprocessed()], vec![preprocessed(), raw]] {
assert_eq!(
from_proto(media).unwrap_err().code(),
tonic::Code::InvalidArgument
);
}
}
#[test]
fn protobuf_features_require_modality_inline_kwargs_and_a_matching_mask() {
let mut missing_modality = preprocessed();
missing_modality.modality = 99;
let mut missing_kwargs = preprocessed();
let Some(pb::media_item::Source::Features(feature)) = &mut missing_kwargs.source else {
unreachable!()
};
feature.kwargs = None;
let mut invalid_mask = preprocessed();
let Some(pb::media_item::Source::Features(feature)) = &mut invalid_mask.source else {
unreachable!()
};
feature.is_embed = vec![true, false];
for media in [missing_modality, missing_kwargs, invalid_mask] {
let error = from_proto(vec![media]).unwrap_err();
assert_eq!(error.code(), tonic::Code::InvalidArgument);
assert!(error.message().starts_with("media[0]:"));
}
}
#[test]
fn aggregate_payload_limit_is_checked_before_decoding_items() {
let mut item = preprocessed();
let Some(pb::media_item::Source::Features(feature)) = &mut item.source else {
unreachable!()
};
feature.kwargs = Some(vec![0; MAX_INLINE_MM_BYTES / 2 + 1]);
let error = from_proto(vec![item.clone(), item]).unwrap_err();
assert_eq!(error.code(), tonic::Code::ResourceExhausted);
}
#[test]
fn kwargs_decoder_rejects_trailing_truncated_and_deeply_nested_input() {
let kwargs = BTreeMap::from([(
"pixels".to_owned(),
MmFieldElem {
data: Some(MmKwargValue::Int(1)),
field: MmField::Batched(MmBatchedField { keep_on_cpu: false }),
},
)]);
let valid = rmp_serde::to_vec_named(&kwargs).unwrap();
assert_eq!(decode_inline_mm_kwargs(&valid).unwrap(), kwargs);
let mut trailing = valid.clone();
trailing.push(0);
let mut nested_kwargs = kwargs.clone();
let mut value = MmKwargValue::Int(1);
for _ in 0..64 {
value = MmKwargValue::List(vec![value]);
}
nested_kwargs.get_mut("pixels").unwrap().data = Some(value);
let nested = rmp_serde::to_vec_named(&nested_kwargs).unwrap();
// A declared array length must not drive a proportional allocation.
let huge_array = vec![
0x81, 0xa1, b'x', 0x81, 0xa4, b'd', b'a', b't', b'a', 0xdd, 0xff, 0xff, 0xff, 0xff,
];
for bytes in [&valid[..valid.len() - 1], &trailing, &nested, &huge_array] {
assert!(decode_inline_mm_kwargs(bytes).is_err());
}
}
#[test]
fn kwargs_decode_python_encoder_tensor_and_factory_tuple() {
// MsgpackEncoder(size_threshold=1 << 30), vLLM 6cbb3c154e:
// pixel_values = tensor([1., 2.]), MultiModalFlatField(slices=[slice(0, 2)]).
let bytes = b"\x81\xacpixel_values\x82\xa4data\x93\xa7float32\x91\x02\xd7\x03\
\x00\x00\x80\x3f\x00\x00\x00\x40\
\xa5field\x92\xa4flat\x83\xabkeep_on_cpu\xc2\
\xa6slices\x91\x93\x00\x02\xc0\xa3dim\x00";
let kwargs = decode_inline_mm_kwargs(bytes).unwrap();
let expected = MmFieldElem {
data: Some(MmKwargValue::Tensor(
WireTensor::from_f32(vec![2], vec![1., 2.]).unwrap(),
)),
field: MmField::Flat(MmFlatField {
slices: vec![MmSlice::Slice(SliceSpec {
start: Some(0),
stop: Some(2),
step: None,
})],
dim: 0,
keep_on_cpu: false,
}),
};
assert_eq!(kwargs["pixel_values"], expected);
}
}
+1
View File
@@ -7,6 +7,7 @@ mod control;
mod convert;
mod health;
mod inference;
mod media;
/// Generated protobuf/gRPC types for the `vllm` package.
pub use vllm_proto as pb;
+257 -1
View File
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::collections::BTreeMap;
use std::fs;
use std::future::Future;
use std::io;
@@ -23,6 +24,7 @@ use tonic_health::pb::health_check_response::ServingStatus as HealthServingStatu
use tonic_health::pb::health_client::HealthClient;
use tonic_health::server::health_reporter;
use tower::service_fn;
use vllm_chat::multimodal::{MmLimitPerPrompt, MmLimitSpec, MmModality};
use vllm_chat::{
ChatBackend, ChatLlm, ChatRenderer, ChatRequest, ChatTextBackend, DefaultChatOutputProcessor,
DynChatOutputProcessor, DynChatRenderer, NewChatOutputProcessorOptions, RenderedPrompt,
@@ -32,12 +34,17 @@ use vllm_engine_core_client::mock_engine::{
default_ready_response,
};
use vllm_engine_core_client::protocol::decode_value;
use vllm_engine_core_client::protocol::dtype::TensorDtype;
use vllm_engine_core_client::protocol::handshake::{EngineCoreReadyResponse, KvEventsConfig};
use vllm_engine_core_client::protocol::multimodal::{
MmBatchedField, MmField, MmFieldElem, MmKwargValue,
};
use vllm_engine_core_client::protocol::output::{
EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, RequestBatchOutputs,
UtilityCallOutput,
};
use vllm_engine_core_client::protocol::request::EngineCoreRequest;
use vllm_engine_core_client::protocol::tensor::WireTensor;
use vllm_engine_core_client::protocol::utility::{UtilityOutput, UtilityResultEnvelope};
use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task_with_ready};
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId, TransportMode};
@@ -292,6 +299,12 @@ const QWEN_IMAGE_TOKEN_ID: u32 = 151655;
const TINY_PNG_DATA_URI: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=";
fn multimodal_backend() -> Arc<dyn ChatTextBackend> {
multimodal_backend_with_limits(Default::default())
}
fn multimodal_backend_with_limits(
limit_mm_per_prompt: MmLimitPerPrompt,
) -> Arc<dyn ChatTextBackend> {
let config_path = std::env::temp_dir().join(format!(
"vllm-grpc-qwen-config-{}.json",
uuid::Uuid::new_v4()
@@ -309,7 +322,7 @@ fn multimodal_backend() -> Arc<dyn ChatTextBackend> {
..Default::default()
},
Arc::new(TestTokenizer::new().with_regular_token("<|image_pad|>", QWEN_IMAGE_TOKEN_ID)),
Default::default(),
limit_mm_per_prompt,
)
.expect("load multimodal info")
.expect("qwen multimodal info is registered");
@@ -877,6 +890,249 @@ async fn unary_generate_prepares_multimodal_input_for_engine_core() {
server_task.abort();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn generate_forwards_preprocessed_multimodal_features() {
for stream in [false, true] {
let kwargs = BTreeMap::from([(
"pixel_values".to_string(),
MmFieldElem {
data: Some(MmKwargValue::Tensor(WireTensor::from_raw(
TensorDtype::U8,
vec![3],
vec![1, 2, 3],
))),
field: MmField::Batched(MmBatchedField { keep_on_cpu: false }),
},
)]);
let encoded_kwargs = rmp_serde::to_vec_named(&kwargs).expect("encode multimodal kwargs");
let expected_kwargs = kwargs.clone();
let (inference_service, control_service, engine_health, engine_task) =
setup_grpc_service_with_backend(
b"engine-grpc-preprocessed-multimodal",
default_stream_output_specs(),
multimodal_backend(),
move |request| {
assert_eq!(
request.prompt_token_ids.as_deref(),
Some(&[11, 99, 99, 12][..])
);
let features = request.mm_features.as_ref().expect("multimodal features");
assert_eq!(features.len(), 1);
let feature = &features[0];
assert_eq!(feature.modality.as_str(), "image");
assert_eq!(feature.identifier, "image-hash-a");
assert_eq!(feature.mm_hash.as_deref(), Some("image-hash-a"));
assert_eq!(feature.mm_position.offset, 1);
assert_eq!(feature.mm_position.length, 2);
assert_eq!(feature.data.as_ref(), Some(&expected_kwargs));
assert_eq!(
feature.mm_position.is_embed,
Some(WireTensor::from_bool(vec![2], vec![true, false]).unwrap())
);
},
)
.await;
let (channel, server_task) = start_grpc_test_server(
inference_service,
control_service,
engine_health,
tokio_util::sync::CancellationToken::new(),
)
.await;
let mut client = InferenceClient::new(channel);
let request = pb::GenerateRequest {
request_id: "test-preprocessed-multimodal".to_string(),
model: "test-model".to_string(),
prompt: Some(pb::generate_request::Prompt::TokenIds(pb::TokenIds {
ids: vec![11, 99, 99, 12],
})),
media: vec![pb::MediaItem {
modality: pb::Modality::Image as i32,
source: Some(pb::media_item::Source::Features(
pb::PreprocessedMediaFeatures {
kwargs: Some(encoded_kwargs),
identifier: "image-hash-a".to_string(),
offset: 1,
length: 2,
mm_hash: Some("image-hash-a".to_string()),
is_embed: vec![true, false],
},
)),
mime_type: String::new(),
uuid: String::new(),
}],
stopping: Some(pb::StoppingCriteria {
max_new_tokens: 10,
..Default::default()
}),
..Default::default()
};
if stream {
let mut response =
client.generate_stream(request).await.expect("preprocessed stream").into_inner();
while response.message().await.expect("stream message").is_some() {}
} else {
client.generate(request).await.expect("preprocessed generate");
}
engine_task.await.expect("mock engine task");
server_task.abort();
}
}
fn preprocessed_grpc_media(
modality: pb::Modality,
identifier: &str,
offset: u64,
length: u64,
) -> pb::MediaItem {
let kwargs = BTreeMap::from([(
"pixel_values".to_string(),
MmFieldElem {
data: Some(MmKwargValue::Tensor(WireTensor::from_raw(
TensorDtype::U8,
vec![1],
vec![1],
))),
field: MmField::Batched(MmBatchedField { keep_on_cpu: false }),
},
)]);
pb::MediaItem {
modality: modality as i32,
source: Some(pb::media_item::Source::Features(
pb::PreprocessedMediaFeatures {
kwargs: Some(rmp_serde::to_vec_named(&kwargs).expect("encode kwargs")),
identifier: identifier.to_string(),
offset,
length,
mm_hash: Some(identifier.to_string()),
is_embed: Vec::new(),
},
)),
mime_type: String::new(),
uuid: String::new(),
}
}
fn preprocessed_grpc_request(media: Vec<pb::MediaItem>) -> pb::GenerateRequest {
pb::GenerateRequest {
request_id: "test-preprocessed-validation".to_string(),
model: "test-model".to_string(),
prompt: Some(pb::generate_request::Prompt::TokenIds(pb::TokenIds {
ids: vec![11, 99, 99, 12],
})),
media,
stopping: Some(pb::StoppingCriteria {
max_new_tokens: 10,
..Default::default()
}),
..Default::default()
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn unary_generate_rejects_preprocessed_features_for_text_only_model() {
let (inference_service, control_service, engine_health, _engine_task) =
setup_grpc_service_with_backend(
b"engine-grpc-preprocessed-text-only",
default_stream_output_specs(),
Arc::new(FakeTextBackend),
|_| {},
)
.await;
let (channel, server_task) = start_grpc_test_server(
inference_service,
control_service,
engine_health,
tokio_util::sync::CancellationToken::new(),
)
.await;
let mut client = InferenceClient::new(channel);
let status = client
.generate(preprocessed_grpc_request(vec![preprocessed_grpc_media(
pb::Modality::Image,
"image-a",
1,
2,
)]))
.await
.expect_err("text-only models must reject preprocessed features");
assert_eq!(status.code(), tonic::Code::InvalidArgument);
server_task.abort();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn generate_validates_preprocessed_model_limits_and_prompt() {
let limits = MmLimitPerPrompt::from([(MmModality::Image, MmLimitSpec::Count(1))]);
let (inference_service, control_service, engine_health, _engine_task) =
setup_grpc_service_with_backend(
b"engine-grpc-preprocessed-limit",
default_stream_output_specs(),
multimodal_backend_with_limits(limits),
|_| {},
)
.await;
let (channel, server_task) = start_grpc_test_server(
inference_service,
control_service,
engine_health,
tokio_util::sync::CancellationToken::new(),
)
.await;
let mut client = InferenceClient::new(channel);
let over_limit = preprocessed_grpc_request(vec![
preprocessed_grpc_media(pb::Modality::Image, "image-a", 0, 1),
preprocessed_grpc_media(pb::Modality::Image, "image-b", 2, 1),
]);
let bad_range = preprocessed_grpc_request(vec![preprocessed_grpc_media(
pb::Modality::Image,
"image",
3,
2,
)]);
let unsupported = preprocessed_grpc_request(vec![preprocessed_grpc_media(
pb::Modality::Audio,
"audio",
0,
1,
)]);
let mut text = preprocessed_grpc_request(vec![preprocessed_grpc_media(
pb::Modality::Image,
"image",
0,
1,
)]);
let mut truncated = text.clone();
text.prompt = Some(pb::generate_request::Prompt::Text("hello".to_owned()));
truncated.truncate_prompt_tokens = 1;
let empty_identifier =
preprocessed_grpc_request(vec![preprocessed_grpc_media(pb::Modality::Image, "", 0, 1)]);
for request in [
over_limit,
bad_range,
unsupported,
text,
truncated,
empty_identifier,
] {
let status = client.generate(request.clone()).await.expect_err("invalid multimodal input");
assert_eq!(status.code(), tonic::Code::InvalidArgument);
let status = client
.generate_stream(request)
.await
.expect_err("invalid streaming multimodal input");
assert_eq!(status.code(), tonic::Code::InvalidArgument);
}
server_task.abort();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn streaming_generate_rejects_text_prompt_with_media() {