[Rust Frontend][Multimodal] Accept preprocessed multimodal gRPC features (#55047)
Signed-off-by: Biswa Panda <biswa.panda@gmail.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Bugen Zhao
Codex
parent
a0844fa6c6
commit
31759ccb6d
Generated
+1
-1
@@ -5940,7 +5940,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "vllm-proto"
|
||||
version = "0.1.0"
|
||||
version = "0.2.0"
|
||||
dependencies = [
|
||||
"prost",
|
||||
"prost-types",
|
||||
|
||||
+1
-1
@@ -147,7 +147,7 @@ vllm-llm = { path = "src/llm" }
|
||||
vllm-managed-engine = { path = "src/managed-engine" }
|
||||
vllm-metrics = { path = "src/metrics" }
|
||||
vllm-parser = { path = "src/parser" }
|
||||
vllm-proto = { version = "0.1.0", path = "proto" }
|
||||
vllm-proto = { version = "0.2.0", path = "proto" }
|
||||
vllm-server = { path = "src/server" }
|
||||
vllm-text = { path = "src/text" }
|
||||
vllm-tokenizer = { path = "src/tokenizer" }
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "vllm-proto"
|
||||
version = "0.1.0"
|
||||
version = "0.2.0"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
description = "Protobuf messages and gRPC bindings for the vLLM Rust frontend"
|
||||
|
||||
@@ -17,7 +17,7 @@ Repository setup requires a `BUF_TOKEN` GitHub Actions secret with permission to
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
vllm-proto = "0.1"
|
||||
vllm-proto = "0.2"
|
||||
```
|
||||
|
||||
For example, import `vllm_proto::inference_client::InferenceClient` or
|
||||
|
||||
@@ -228,7 +228,20 @@ message MediaItem {
|
||||
string url = 2; // http:// or https://
|
||||
string data_uri = 3; // data:
|
||||
bytes raw_bytes = 4;
|
||||
PreprocessedMediaFeatures features = 7;
|
||||
}
|
||||
string mime_type = 5;
|
||||
string uuid = 6;
|
||||
}
|
||||
|
||||
// Engine-ready multimodal features produced by a version-compatible frontend.
|
||||
// `kwargs` is the MessagePack wire representation of one MmKwargsItem.
|
||||
message PreprocessedMediaFeatures {
|
||||
// Required inline payload. Cache-only references are not accepted.
|
||||
optional bytes kwargs = 1;
|
||||
string identifier = 2;
|
||||
uint64 offset = 3;
|
||||
uint64 length = 4;
|
||||
optional string mm_hash = 5;
|
||||
repeated bool is_embed = 6;
|
||||
}
|
||||
|
||||
@@ -27,6 +27,8 @@ pub enum Error {
|
||||
UnsupportedModality { modality: String },
|
||||
#[error("At most {limit} {modality}(s) may be provided in one prompt.")]
|
||||
MmLimitExceeded { modality: String, limit: usize },
|
||||
#[error("invalid inline multimodal features: {message}")]
|
||||
InvalidPreprocessedMultimodal { message: String },
|
||||
#[error("multimodal preprocessing error: {0}")]
|
||||
Multimodal(#[message] String),
|
||||
#[error("{kind} parsing is not available for model `{model_id}`")]
|
||||
@@ -106,6 +108,7 @@ impl Error {
|
||||
Self::UnsupportedMultimodalRenderer
|
||||
| Self::UnsupportedMultimodalContent(_)
|
||||
| Self::UnsupportedModality { .. }
|
||||
| Self::InvalidPreprocessedMultimodal { .. }
|
||||
| Self::MmLimitExceeded { .. } => true,
|
||||
|
||||
_ => false,
|
||||
|
||||
@@ -130,7 +130,7 @@ impl ChatRequestProcessor {
|
||||
/// Prepare media for an already-tokenized request.
|
||||
async fn prepare_media(
|
||||
&self,
|
||||
media: Vec<MediaContentPart>,
|
||||
media: multimodal::MultimodalInput,
|
||||
token_ids: &mut Vec<u32>,
|
||||
) -> Result<Option<MmFeatures>> {
|
||||
if media.is_empty() {
|
||||
@@ -140,8 +140,15 @@ impl ChatRequestProcessor {
|
||||
.backend
|
||||
.multimodal_model_info()
|
||||
.ok_or(Error::UnsupportedMultimodalRenderer)?;
|
||||
let model_dtype = self.model_dtype.ok_or(Error::UnsupportedMultimodalRenderer)?;
|
||||
let features = info.prepare_multimodal(media, token_ids, model_dtype).await?;
|
||||
let features = match media {
|
||||
multimodal::MultimodalInput::Raw(parts) => {
|
||||
let model_dtype = self.model_dtype.ok_or(Error::UnsupportedMultimodalRenderer)?;
|
||||
info.prepare_multimodal(parts, token_ids, model_dtype).await?
|
||||
}
|
||||
multimodal::MultimodalInput::Preprocessed(features) => {
|
||||
info.prepare_preprocessed(features, token_ids.len())?
|
||||
}
|
||||
};
|
||||
Ok(Some(features))
|
||||
}
|
||||
|
||||
@@ -286,12 +293,14 @@ impl ChatLlm {
|
||||
}
|
||||
|
||||
/// Prepare media for an already-tokenized request.
|
||||
/// Raw content is preprocessed; inline features are checked against model
|
||||
/// capabilities, modality limits, and the final prompt positions.
|
||||
pub async fn prepare_media(
|
||||
&self,
|
||||
media: Vec<MediaContentPart>,
|
||||
media: impl Into<multimodal::MultimodalInput>,
|
||||
token_ids: &mut Vec<u32>,
|
||||
) -> Result<Option<MmFeatures>> {
|
||||
self.processor.prepare_media(media, token_ids).await
|
||||
self.processor.prepare_media(media.into(), token_ids).await
|
||||
}
|
||||
|
||||
/// Effective tool-call parser name for this model, if parsing is enabled.
|
||||
|
||||
@@ -39,11 +39,14 @@ use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest};
|
||||
mod audio;
|
||||
mod expand;
|
||||
mod image;
|
||||
mod input;
|
||||
mod item;
|
||||
mod preprocessed;
|
||||
mod tensor;
|
||||
mod video;
|
||||
|
||||
use self::expand::expand_prompt_token_ids;
|
||||
pub use self::input::MultimodalInput;
|
||||
|
||||
/// Resolved multimodal support for one loaded model.
|
||||
#[derive(Clone)]
|
||||
@@ -654,11 +657,16 @@ impl MultimodalModelInfo {
|
||||
///
|
||||
/// Modalities without a configured count are unlimited.
|
||||
fn validate_mm_limits(&self, media_parts: &[MediaContentPart]) -> Result<()> {
|
||||
self.validate_modality_limits(media_parts.iter().filter_map(media_part_limit_modality))
|
||||
}
|
||||
|
||||
fn validate_modality_limits(
|
||||
&self,
|
||||
modalities: impl IntoIterator<Item = MmModality>,
|
||||
) -> Result<()> {
|
||||
let mut counts: HashMap<MmModality, usize> = HashMap::new();
|
||||
for part in media_parts {
|
||||
if let Some(modality) = media_part_limit_modality(part) {
|
||||
*counts.entry(modality).or_default() += 1;
|
||||
}
|
||||
for modality in modalities {
|
||||
*counts.entry(modality).or_default() += 1;
|
||||
}
|
||||
|
||||
for (modality, count) in counts {
|
||||
@@ -677,6 +685,32 @@ impl MultimodalModelInfo {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Validate inline storage, batching, and placeholder ranges, then check
|
||||
/// this model's supported modalities and item-count limits.
|
||||
///
|
||||
/// `prompt_len` must include all expanded multimodal placeholders.
|
||||
pub(crate) fn prepare_preprocessed(
|
||||
&self,
|
||||
mut features: MmFeatures,
|
||||
prompt_len: usize,
|
||||
) -> Result<MmFeatures> {
|
||||
preprocessed::validate_features(&mut features, prompt_len)?;
|
||||
for feature in &features {
|
||||
let supported = match feature.modality {
|
||||
MmModality::Image => self.image.is_some(),
|
||||
MmModality::Video => self.video.is_some(),
|
||||
MmModality::Audio => self.audio.is_some(),
|
||||
};
|
||||
if !supported {
|
||||
return Err(Error::UnsupportedModality {
|
||||
modality: feature.modality.as_str().to_owned(),
|
||||
});
|
||||
}
|
||||
}
|
||||
self.validate_modality_limits(features.iter().map(|feature| feature.modality))?;
|
||||
Ok(features)
|
||||
}
|
||||
|
||||
/// Run media fetch, per-modality preprocessing, prompt expansion, and
|
||||
/// feature build.
|
||||
///
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use llm_multimodal::MediaContentPart;
|
||||
use vllm_engine_core_client::protocol::multimodal::MmFeatures;
|
||||
|
||||
/// Media preparation stage supplied by a frontend handler.
|
||||
#[derive(Debug)]
|
||||
pub enum MultimodalInput {
|
||||
/// Raw media to fetch and preprocess, expanding prompt placeholders as needed.
|
||||
Raw(Vec<MediaContentPart>),
|
||||
/// Preprocessed features to validate against storage, batching, model, and prompt constraints.
|
||||
Preprocessed(MmFeatures),
|
||||
}
|
||||
|
||||
impl MultimodalInput {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
match self {
|
||||
Self::Raw(parts) => parts.is_empty(),
|
||||
Self::Preprocessed(features) => features.is_empty(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Vec<MediaContentPart>> for MultimodalInput {
|
||||
fn from(parts: Vec<MediaContentPart>) -> Self {
|
||||
Self::Raw(parts)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,274 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use vllm_engine_core_client::protocol::dtype::TensorDtype;
|
||||
use vllm_engine_core_client::protocol::multimodal::{
|
||||
MmFeatureSpec, MmField, MmKwargValue, MmKwargsItem,
|
||||
};
|
||||
|
||||
use crate::error::{
|
||||
Error, Result, bail_invalid_preprocessed_multimodal, invalid_preprocessed_multimodal,
|
||||
};
|
||||
|
||||
const MAX_DEPTH: usize = 32;
|
||||
|
||||
/// Validate inline storage, placeholder ranges, and batching compatibility,
|
||||
/// then stably sort features by their prompt offsets.
|
||||
/// Encoder cache identifiers are supplied by the producer, which must bind
|
||||
/// them to the feature content.
|
||||
pub(super) fn validate_features(features: &mut [MmFeatureSpec], prompt_len: usize) -> Result<()> {
|
||||
let mut fields = HashMap::new();
|
||||
for feature in features.iter() {
|
||||
if feature.identifier.is_empty() {
|
||||
bail_invalid_preprocessed_multimodal!("feature identifier must be nonempty");
|
||||
}
|
||||
let data = feature.data.as_ref().ok_or_else(|| {
|
||||
invalid_preprocessed_multimodal!(
|
||||
"inline data is required; cache-only features are unsupported"
|
||||
)
|
||||
})?;
|
||||
validate_kwargs(data)?;
|
||||
for (key, elem) in data {
|
||||
if let Some(previous) = fields.insert((feature.modality, key), &elem.field)
|
||||
&& !same_batching(previous, &elem.field)
|
||||
{
|
||||
bail_invalid_preprocessed_multimodal!(
|
||||
"incompatible batching for {} field {key:?}",
|
||||
feature.modality.as_str()
|
||||
);
|
||||
}
|
||||
}
|
||||
let position = &feature.mm_position;
|
||||
if position.length == 0 || position.offset.checked_add(position.length).is_none() {
|
||||
bail_invalid_preprocessed_multimodal!(
|
||||
"placeholder range must have a positive length and fit in usize"
|
||||
);
|
||||
}
|
||||
if position.offset + position.length > prompt_len {
|
||||
bail_invalid_preprocessed_multimodal!(
|
||||
"{} placeholder range exceeds prompt token IDs",
|
||||
feature.modality.as_str()
|
||||
);
|
||||
}
|
||||
if let Some(mask) = &position.is_embed {
|
||||
if mask.dtype != TensorDtype::Bool || mask.shape != [position.length] {
|
||||
bail_invalid_preprocessed_multimodal!(
|
||||
"is_embed must be a boolean tensor of shape [placeholder length]"
|
||||
);
|
||||
}
|
||||
mask.validate_inline()
|
||||
.map_err(|message| Error::InvalidPreprocessedMultimodal { message })?;
|
||||
}
|
||||
}
|
||||
// Mirror the Python frontend (`argsort_mm_positions`): features are
|
||||
// ordered by their placeholder position in the prompt.
|
||||
features.sort_by_key(|feature| feature.mm_position.offset);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_kwargs(kwargs: &MmKwargsItem) -> Result<()> {
|
||||
if kwargs.is_empty() {
|
||||
bail_invalid_preprocessed_multimodal!("kwargs must contain at least one field");
|
||||
}
|
||||
for (key, elem) in kwargs {
|
||||
if key.is_empty() {
|
||||
bail_invalid_preprocessed_multimodal!("kwargs field name must be nonempty");
|
||||
}
|
||||
let data = elem.data.as_ref().ok_or_else(|| {
|
||||
invalid_preprocessed_multimodal!("field {key:?} requires inline data")
|
||||
})?;
|
||||
validate_value(data, 0)?;
|
||||
if let MmField::Flat(field) = &elem.field {
|
||||
match data {
|
||||
MmKwargValue::Tensor(tensor) => {
|
||||
let rank = tensor.shape.len() as i32;
|
||||
if rank == 0 || field.dim < -rank || field.dim >= rank {
|
||||
bail_invalid_preprocessed_multimodal!(
|
||||
"flat field {key:?} dim is outside the tensor rank"
|
||||
);
|
||||
}
|
||||
}
|
||||
MmKwargValue::List(_) if field.dim == 0 => {}
|
||||
_ => bail_invalid_preprocessed_multimodal!(
|
||||
"flat field {key:?} requires a tensor or a list with dim=0"
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_value(value: &MmKwargValue, depth: usize) -> Result<()> {
|
||||
if depth > MAX_DEPTH {
|
||||
bail_invalid_preprocessed_multimodal!("kwargs nesting exceeds {MAX_DEPTH} levels");
|
||||
}
|
||||
match value {
|
||||
MmKwargValue::Tensor(tensor) => tensor
|
||||
.validate_inline()
|
||||
.map_err(|message| Error::InvalidPreprocessedMultimodal { message }),
|
||||
MmKwargValue::List(values) => {
|
||||
for value in values {
|
||||
validate_value(value, depth + 1)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
MmKwargValue::Int(_) | MmKwargValue::Float(_) => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
fn same_batching(left: &MmField, right: &MmField) -> bool {
|
||||
// Slices and batch_size describe the producer's original batch. Python
|
||||
// reduce_data combines the already-split item data without using them.
|
||||
match (left, right) {
|
||||
(MmField::Batched(a), MmField::Batched(b)) => a == b,
|
||||
(MmField::Flat(a), MmField::Flat(b)) => a.dim == b.dim && a.keep_on_cpu == b.keep_on_cpu,
|
||||
(MmField::Shared(a), MmField::Shared(b)) => a.keep_on_cpu == b.keep_on_cpu,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use vllm_engine_core_client::protocol::multimodal::{
|
||||
MmBatchedField, MmFieldElem, MmFlatField, MmModality, MmSharedField, MmSlice,
|
||||
PlaceholderRange, SliceSpec,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::tensor::{WireArrayData, WireTensor};
|
||||
|
||||
use super::*;
|
||||
use crate::multimodal::tests::qwen3_vl_info;
|
||||
|
||||
fn feature(offset: usize, field: MmField) -> MmFeatureSpec {
|
||||
MmFeatureSpec {
|
||||
data: Some(BTreeMap::from([(
|
||||
"pixels".to_owned(),
|
||||
MmFieldElem {
|
||||
data: Some(MmKwargValue::Tensor(
|
||||
WireTensor::from_f32(vec![2], vec![1., 2.]).unwrap(),
|
||||
)),
|
||||
field,
|
||||
},
|
||||
)])),
|
||||
modality: MmModality::Image,
|
||||
identifier: format!("image-{offset}"),
|
||||
mm_position: PlaceholderRange {
|
||||
offset,
|
||||
length: 2,
|
||||
is_embed: None,
|
||||
},
|
||||
mm_hash: Some("processor-hash".to_owned()),
|
||||
}
|
||||
}
|
||||
|
||||
fn batched() -> MmField {
|
||||
MmField::Batched(MmBatchedField { keep_on_cpu: false })
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inline_features_preserve_overlaps_and_stably_sort_prompt_positions() {
|
||||
let info = qwen3_vl_info();
|
||||
let mut a = feature(2, batched());
|
||||
a.mm_position.is_embed = Some(WireTensor::from_bool(vec![2], vec![true, false]).unwrap());
|
||||
let b = feature(1, batched());
|
||||
let mut c = feature(2, batched());
|
||||
c.modality = MmModality::Video;
|
||||
let expected = vec![b.clone(), a.clone(), c.clone()];
|
||||
let features = info.prepare_preprocessed(vec![a, b, c], 4).unwrap();
|
||||
assert_eq!(features, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn independently_preprocessed_items_keep_their_original_batch_metadata() {
|
||||
let info = qwen3_vl_info();
|
||||
let flat = |length| {
|
||||
MmField::Flat(MmFlatField {
|
||||
slices: vec![MmSlice::Slice(SliceSpec {
|
||||
start: Some(0),
|
||||
stop: Some(length),
|
||||
step: None,
|
||||
})],
|
||||
dim: 0,
|
||||
keep_on_cpu: false,
|
||||
})
|
||||
};
|
||||
let shared = |batch_size| {
|
||||
MmField::Shared(MmSharedField {
|
||||
batch_size,
|
||||
keep_on_cpu: true,
|
||||
})
|
||||
};
|
||||
for (a, b) in [(flat(2), flat(7)), (shared(1), shared(3))] {
|
||||
let features = vec![feature(0, a), feature(2, b)];
|
||||
let result = info.prepare_preprocessed(features.clone(), 4).unwrap();
|
||||
assert_eq!(result, features);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inline_features_reject_cache_references_and_invalid_tensor_storage() {
|
||||
let info = qwen3_vl_info();
|
||||
let original = feature(0, batched());
|
||||
let mut cached = original.clone();
|
||||
cached.data = None;
|
||||
assert!(info.prepare_preprocessed(vec![cached], 4).is_err());
|
||||
let mut field_cached = original.clone();
|
||||
field_cached.data.as_mut().unwrap().get_mut("pixels").unwrap().data = None;
|
||||
assert!(info.prepare_preprocessed(vec![field_cached], 4).is_err());
|
||||
for tensor in [
|
||||
WireTensor {
|
||||
dtype: TensorDtype::F32,
|
||||
shape: vec![2],
|
||||
data: WireArrayData::AuxIndex(1),
|
||||
},
|
||||
WireTensor::from_raw(TensorDtype::F32, vec![2], vec![0; 4]),
|
||||
WireTensor::from_raw(TensorDtype::F32, vec![usize::MAX, 2], vec![]),
|
||||
] {
|
||||
let mut item = original.clone();
|
||||
item.data.as_mut().unwrap().get_mut("pixels").unwrap().data =
|
||||
Some(MmKwargValue::Tensor(tensor));
|
||||
assert!(info.prepare_preprocessed(vec![item], 4).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inline_features_validate_placeholder_and_mask_bounds() {
|
||||
let info = qwen3_vl_info();
|
||||
for (offset, length) in [(usize::MAX, 2), (0, 0)] {
|
||||
let mut item = feature(offset, batched());
|
||||
item.mm_position.length = length;
|
||||
assert!(info.prepare_preprocessed(vec![item], 4).is_err());
|
||||
}
|
||||
assert!(info.prepare_preprocessed(vec![feature(2, batched())], 3).is_err());
|
||||
for mask in [
|
||||
WireTensor::from_bool(vec![1], vec![true]).unwrap(),
|
||||
WireTensor::from_u32(vec![2], vec![1, 0]).unwrap(),
|
||||
] {
|
||||
let mut item = feature(0, batched());
|
||||
item.mm_position.is_embed = Some(mask);
|
||||
assert!(info.prepare_preprocessed(vec![item], 4).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inline_features_reject_incompatible_batching_and_flat_axes() {
|
||||
let info = qwen3_vl_info();
|
||||
let flat = MmField::Flat(MmFlatField {
|
||||
slices: vec![],
|
||||
dim: 2,
|
||||
keep_on_cpu: false,
|
||||
});
|
||||
assert!(info.prepare_preprocessed(vec![feature(0, flat)], 4).is_err());
|
||||
let shared = MmField::Shared(MmSharedField {
|
||||
batch_size: 1,
|
||||
keep_on_cpu: false,
|
||||
});
|
||||
assert!(
|
||||
info.prepare_preprocessed(vec![feature(0, batched()), feature(2, shared)], 4)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,7 @@ libc.workspace = true
|
||||
llm-multimodal.workspace = true
|
||||
openssl.workspace = true
|
||||
prost-types.workspace = true
|
||||
rmp-serde.workspace = true
|
||||
rmpv.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
@@ -58,7 +59,6 @@ async-openai = { workspace = true, features = ["full"] }
|
||||
bytes.workspace = true
|
||||
clap.workspace = true
|
||||
expect-test.workspace = true
|
||||
rmp-serde.workspace = true
|
||||
serial_test.workspace = true
|
||||
tempfile.workspace = true
|
||||
tokio = { workspace = true, features = ["test-util"] }
|
||||
|
||||
@@ -5,9 +5,7 @@
|
||||
//! request/response types.
|
||||
|
||||
use tonic::Status;
|
||||
use url::Url;
|
||||
use uuid::Uuid;
|
||||
use vllm_chat::MediaContentPart;
|
||||
use vllm_engine_core_client::protocol::output::StopReason;
|
||||
use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsParams;
|
||||
use vllm_text::{
|
||||
@@ -17,95 +15,6 @@ use vllm_text::{
|
||||
|
||||
use super::pb;
|
||||
|
||||
pub fn media_parts_from_request(
|
||||
media: Vec<pb::MediaItem>,
|
||||
) -> Result<Vec<MediaContentPart>, Status> {
|
||||
let mut parts = Vec::with_capacity(media.len());
|
||||
for (index, item) in media.into_iter().enumerate() {
|
||||
let modality = item.modality();
|
||||
if modality == pb::Modality::Unspecified {
|
||||
return Err(Status::invalid_argument(format!(
|
||||
"media[{index}].modality is required"
|
||||
)));
|
||||
}
|
||||
let uuid = (!item.uuid.is_empty()).then_some(item.uuid);
|
||||
let mime_type = (!item.mime_type.is_empty()).then_some(item.mime_type);
|
||||
let source = item.source.ok_or_else(|| {
|
||||
Status::invalid_argument(format!("media[{index}].source is required"))
|
||||
})?;
|
||||
match &source {
|
||||
pb::media_item::Source::Url(url) => {
|
||||
validate_media_uri(index, "url", url, &["http", "https"])?;
|
||||
}
|
||||
pb::media_item::Source::DataUri(uri) => {
|
||||
validate_media_uri(index, "data_uri", uri, &["data"])?;
|
||||
}
|
||||
pb::media_item::Source::RawBytes(_) => {}
|
||||
}
|
||||
let part = match (modality, source) {
|
||||
(
|
||||
pb::Modality::Image,
|
||||
pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url),
|
||||
) => MediaContentPart::ImageUrl {
|
||||
url,
|
||||
detail: None,
|
||||
uuid,
|
||||
},
|
||||
(pb::Modality::Image, pb::media_item::Source::RawBytes(data)) => {
|
||||
MediaContentPart::ImageData {
|
||||
data,
|
||||
mime_type,
|
||||
uuid,
|
||||
detail: None,
|
||||
}
|
||||
}
|
||||
(
|
||||
pb::Modality::Video,
|
||||
pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url),
|
||||
) => MediaContentPart::VideoUrl { url, uuid },
|
||||
(pb::Modality::Video, pb::media_item::Source::RawBytes(data)) => {
|
||||
MediaContentPart::VideoData {
|
||||
data,
|
||||
mime_type,
|
||||
uuid,
|
||||
}
|
||||
}
|
||||
(
|
||||
pb::Modality::Audio,
|
||||
pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url),
|
||||
) => MediaContentPart::AudioUrl { url, uuid },
|
||||
(pb::Modality::Audio, pb::media_item::Source::RawBytes(data)) => {
|
||||
MediaContentPart::AudioData {
|
||||
data,
|
||||
mime_type,
|
||||
uuid,
|
||||
}
|
||||
}
|
||||
(pb::Modality::Unspecified, _) => unreachable!("modality validated above"),
|
||||
};
|
||||
parts.push(part);
|
||||
}
|
||||
Ok(parts)
|
||||
}
|
||||
|
||||
fn validate_media_uri(
|
||||
index: usize,
|
||||
field: &str,
|
||||
value: &str,
|
||||
allowed_schemes: &[&str],
|
||||
) -> Result<(), Status> {
|
||||
let uri = Url::parse(value).map_err(|_| {
|
||||
Status::invalid_argument(format!("media[{index}].{field} is not a valid URI"))
|
||||
})?;
|
||||
if !allowed_schemes.contains(&uri.scheme()) {
|
||||
return Err(Status::invalid_argument(format!(
|
||||
"media[{index}].{field} must use the {} scheme",
|
||||
allowed_schemes.join(" or ")
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ========================================================================================
|
||||
// Request conversion
|
||||
// ========================================================================================
|
||||
|
||||
@@ -149,19 +149,23 @@ impl InferenceServiceImpl {
|
||||
})?);
|
||||
}
|
||||
|
||||
let media = convert::media_parts_from_request(media)?;
|
||||
let media = super::media::from_proto(media)?;
|
||||
if !media.is_empty() {
|
||||
let Prompt::TokenIds(mut token_ids) = text_request.prompt else {
|
||||
return Err(Status::invalid_argument(
|
||||
"multimodal gRPC requests must provide token_ids input",
|
||||
));
|
||||
};
|
||||
let mm_features = self
|
||||
.state
|
||||
.chat
|
||||
.prepare_media(media, &mut token_ids)
|
||||
.await
|
||||
.map_err(|error| Status::internal(error.to_report_string()))?;
|
||||
let mm_features =
|
||||
self.state.chat.prepare_media(media, &mut token_ids).await.map_err(
|
||||
|error| {
|
||||
if error.is_request_validation_error() {
|
||||
Status::invalid_argument(error.to_report_string())
|
||||
} else {
|
||||
Status::internal(error.to_report_string())
|
||||
}
|
||||
},
|
||||
)?;
|
||||
text_request.prompt = Prompt::TokenIds(token_ids);
|
||||
text_request.mm_features = mm_features;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,341 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
//! Protobuf media adaptation into the shared multimodal preparation boundary.
|
||||
|
||||
use std::io::Cursor;
|
||||
|
||||
use serde::Deserialize as _;
|
||||
use thiserror_ext::AsReport as _;
|
||||
use tonic::Status;
|
||||
use url::Url;
|
||||
use vllm_chat::MediaContentPart;
|
||||
use vllm_chat::multimodal::MultimodalInput;
|
||||
use vllm_engine_core_client::protocol::multimodal::{
|
||||
MmFeatureSpec, MmKwargsItem, MmModality, PlaceholderRange,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::tensor::WireTensor;
|
||||
|
||||
use super::pb;
|
||||
|
||||
/// Maximum total serialized kwargs size accepted in one gRPC request.
|
||||
const MAX_INLINE_MM_BYTES: usize = 16 * 1024 * 1024;
|
||||
const MAX_MSGPACK_DEPTH: usize = 32;
|
||||
|
||||
/// Adapt a request containing either raw media or preprocessed features.
|
||||
///
|
||||
/// Require a single input kind and bound the total kwargs size before decoding.
|
||||
/// Shared media preparation validates features, model support, item limits, and prompt bounds.
|
||||
pub(super) fn from_proto(media: Vec<pb::MediaItem>) -> Result<MultimodalInput, Status> {
|
||||
if !media
|
||||
.iter()
|
||||
.any(|item| matches!(item.source, Some(pb::media_item::Source::Features(_))))
|
||||
{
|
||||
return raw_parts_from_proto(media).map(MultimodalInput::Raw);
|
||||
}
|
||||
let mut encoded_bytes = 0usize;
|
||||
for item in &media {
|
||||
let Some(pb::media_item::Source::Features(feature)) = &item.source else {
|
||||
return Err(mixed_media_error());
|
||||
};
|
||||
encoded_bytes = encoded_bytes.saturating_add(feature.kwargs.as_ref().map_or(0, Vec::len));
|
||||
}
|
||||
if encoded_bytes > MAX_INLINE_MM_BYTES {
|
||||
return Err(Status::resource_exhausted(format!(
|
||||
"inline multimodal payload exceeds {MAX_INLINE_MM_BYTES} bytes"
|
||||
)));
|
||||
}
|
||||
let features = media
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(index, item)| {
|
||||
feature_from_proto(item).map_err(|error| {
|
||||
Status::new(error.code(), format!("media[{index}]: {}", error.message()))
|
||||
})
|
||||
})
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
Ok(MultimodalInput::Preprocessed(features))
|
||||
}
|
||||
|
||||
fn feature_from_proto(item: pb::MediaItem) -> Result<MmFeatureSpec, Status> {
|
||||
let modality = match item.modality() {
|
||||
pb::Modality::Image => MmModality::Image,
|
||||
pb::Modality::Video => MmModality::Video,
|
||||
pb::Modality::Audio => MmModality::Audio,
|
||||
pb::Modality::Unspecified => return Err(Status::invalid_argument("modality is required")),
|
||||
};
|
||||
let Some(pb::media_item::Source::Features(feature)) = item.source else {
|
||||
return Err(mixed_media_error());
|
||||
};
|
||||
let bytes = feature.kwargs.ok_or_else(|| {
|
||||
Status::invalid_argument("features.kwargs is required; cache-only features are unsupported")
|
||||
})?;
|
||||
let offset = usize::try_from(feature.offset)
|
||||
.map_err(|_| Status::invalid_argument("features.offset exceeds platform limits"))?;
|
||||
let length = usize::try_from(feature.length)
|
||||
.map_err(|_| Status::invalid_argument("features.length exceeds platform limits"))?;
|
||||
let is_embed = if feature.is_embed.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(
|
||||
WireTensor::from_bool(vec![length], feature.is_embed)
|
||||
.map_err(Status::invalid_argument)?,
|
||||
)
|
||||
};
|
||||
Ok(MmFeatureSpec {
|
||||
modality,
|
||||
data: Some(decode_inline_mm_kwargs(&bytes)?),
|
||||
identifier: feature.identifier,
|
||||
mm_hash: feature.mm_hash,
|
||||
mm_position: PlaceholderRange {
|
||||
offset,
|
||||
length,
|
||||
is_embed,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
/// Decode one inline item using the engine protocol's typed serde schema.
|
||||
fn decode_inline_mm_kwargs(bytes: &[u8]) -> Result<MmKwargsItem, Status> {
|
||||
let mut decoder = rmp_serde::Deserializer::new(Cursor::new(bytes));
|
||||
decoder.set_max_depth(MAX_MSGPACK_DEPTH);
|
||||
let kwargs = MmKwargsItem::deserialize(&mut decoder).map_err(|error| {
|
||||
Status::invalid_argument(format!(
|
||||
"invalid multimodal kwargs MessagePack: {}",
|
||||
error.as_report()
|
||||
))
|
||||
})?;
|
||||
if decoder.position() != bytes.len() as u64 {
|
||||
return Err(Status::invalid_argument(
|
||||
"kwargs contains trailing MessagePack data",
|
||||
));
|
||||
}
|
||||
Ok(kwargs)
|
||||
}
|
||||
|
||||
fn mixed_media_error() -> Status {
|
||||
Status::invalid_argument("raw media and preprocessed media features cannot be mixed")
|
||||
}
|
||||
|
||||
fn raw_parts_from_proto(media: Vec<pb::MediaItem>) -> Result<Vec<MediaContentPart>, Status> {
|
||||
let mut parts = Vec::with_capacity(media.len());
|
||||
for (index, item) in media.into_iter().enumerate() {
|
||||
let modality = item.modality();
|
||||
if modality == pb::Modality::Unspecified {
|
||||
return Err(Status::invalid_argument(format!(
|
||||
"media[{index}].modality is required"
|
||||
)));
|
||||
}
|
||||
let uuid = (!item.uuid.is_empty()).then_some(item.uuid);
|
||||
let mime_type = (!item.mime_type.is_empty()).then_some(item.mime_type);
|
||||
let source = item.source.ok_or_else(|| {
|
||||
Status::invalid_argument(format!("media[{index}].source is required"))
|
||||
})?;
|
||||
match &source {
|
||||
pb::media_item::Source::Url(url) => {
|
||||
validate_media_uri(index, "url", url, &["http", "https"])?;
|
||||
}
|
||||
pb::media_item::Source::DataUri(uri) => {
|
||||
validate_media_uri(index, "data_uri", uri, &["data"])?;
|
||||
}
|
||||
pb::media_item::Source::RawBytes(_) => {}
|
||||
pb::media_item::Source::Features(_) => return Err(mixed_media_error()),
|
||||
}
|
||||
let part = match (modality, source) {
|
||||
(
|
||||
pb::Modality::Image,
|
||||
pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url),
|
||||
) => MediaContentPart::ImageUrl {
|
||||
url,
|
||||
detail: None,
|
||||
uuid,
|
||||
},
|
||||
(pb::Modality::Image, pb::media_item::Source::RawBytes(data)) => {
|
||||
MediaContentPart::ImageData {
|
||||
data,
|
||||
mime_type,
|
||||
uuid,
|
||||
detail: None,
|
||||
}
|
||||
}
|
||||
(
|
||||
pb::Modality::Video,
|
||||
pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url),
|
||||
) => MediaContentPart::VideoUrl { url, uuid },
|
||||
(pb::Modality::Video, pb::media_item::Source::RawBytes(data)) => {
|
||||
MediaContentPart::VideoData {
|
||||
data,
|
||||
mime_type,
|
||||
uuid,
|
||||
}
|
||||
}
|
||||
(
|
||||
pb::Modality::Audio,
|
||||
pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url),
|
||||
) => MediaContentPart::AudioUrl { url, uuid },
|
||||
(pb::Modality::Audio, pb::media_item::Source::RawBytes(data)) => {
|
||||
MediaContentPart::AudioData {
|
||||
data,
|
||||
mime_type,
|
||||
uuid,
|
||||
}
|
||||
}
|
||||
(_, pb::media_item::Source::Features(_)) => return Err(mixed_media_error()),
|
||||
(pb::Modality::Unspecified, _) => unreachable!("modality validated above"),
|
||||
};
|
||||
parts.push(part);
|
||||
}
|
||||
Ok(parts)
|
||||
}
|
||||
|
||||
fn validate_media_uri(
|
||||
index: usize,
|
||||
field: &str,
|
||||
value: &str,
|
||||
allowed_schemes: &[&str],
|
||||
) -> Result<(), Status> {
|
||||
let uri = Url::parse(value).map_err(|_| {
|
||||
Status::invalid_argument(format!("media[{index}].{field} is not a valid URI"))
|
||||
})?;
|
||||
if !allowed_schemes.contains(&uri.scheme()) {
|
||||
return Err(Status::invalid_argument(format!(
|
||||
"media[{index}].{field} must use the {} scheme",
|
||||
allowed_schemes.join(" or ")
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use super::*;
|
||||
use vllm_engine_core_client::protocol::multimodal::{
|
||||
MmBatchedField, MmField, MmFieldElem, MmFlatField, MmKwargValue, MmSlice, SliceSpec,
|
||||
};
|
||||
|
||||
fn preprocessed() -> pb::MediaItem {
|
||||
let kwargs = BTreeMap::from([(
|
||||
"x",
|
||||
MmFieldElem {
|
||||
data: Some(MmKwargValue::Int(1)),
|
||||
field: MmField::Batched(MmBatchedField { keep_on_cpu: false }),
|
||||
},
|
||||
)]);
|
||||
pb::MediaItem {
|
||||
modality: pb::Modality::Image as i32,
|
||||
source: Some(pb::media_item::Source::Features(
|
||||
pb::PreprocessedMediaFeatures {
|
||||
kwargs: Some(rmp_serde::to_vec_named(&kwargs).unwrap()),
|
||||
identifier: "image-id".to_owned(),
|
||||
length: 1,
|
||||
..Default::default()
|
||||
},
|
||||
)),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mixed_media_is_rejected_in_both_orders() {
|
||||
let raw = pb::MediaItem {
|
||||
modality: pb::Modality::Image as i32,
|
||||
source: Some(pb::media_item::Source::RawBytes(vec![1])),
|
||||
..Default::default()
|
||||
};
|
||||
for media in [vec![raw.clone(), preprocessed()], vec![preprocessed(), raw]] {
|
||||
assert_eq!(
|
||||
from_proto(media).unwrap_err().code(),
|
||||
tonic::Code::InvalidArgument
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn protobuf_features_require_modality_inline_kwargs_and_a_matching_mask() {
|
||||
let mut missing_modality = preprocessed();
|
||||
missing_modality.modality = 99;
|
||||
let mut missing_kwargs = preprocessed();
|
||||
let Some(pb::media_item::Source::Features(feature)) = &mut missing_kwargs.source else {
|
||||
unreachable!()
|
||||
};
|
||||
feature.kwargs = None;
|
||||
let mut invalid_mask = preprocessed();
|
||||
let Some(pb::media_item::Source::Features(feature)) = &mut invalid_mask.source else {
|
||||
unreachable!()
|
||||
};
|
||||
feature.is_embed = vec![true, false];
|
||||
for media in [missing_modality, missing_kwargs, invalid_mask] {
|
||||
let error = from_proto(vec![media]).unwrap_err();
|
||||
assert_eq!(error.code(), tonic::Code::InvalidArgument);
|
||||
assert!(error.message().starts_with("media[0]:"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn aggregate_payload_limit_is_checked_before_decoding_items() {
|
||||
let mut item = preprocessed();
|
||||
let Some(pb::media_item::Source::Features(feature)) = &mut item.source else {
|
||||
unreachable!()
|
||||
};
|
||||
feature.kwargs = Some(vec![0; MAX_INLINE_MM_BYTES / 2 + 1]);
|
||||
let error = from_proto(vec![item.clone(), item]).unwrap_err();
|
||||
assert_eq!(error.code(), tonic::Code::ResourceExhausted);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kwargs_decoder_rejects_trailing_truncated_and_deeply_nested_input() {
|
||||
let kwargs = BTreeMap::from([(
|
||||
"pixels".to_owned(),
|
||||
MmFieldElem {
|
||||
data: Some(MmKwargValue::Int(1)),
|
||||
field: MmField::Batched(MmBatchedField { keep_on_cpu: false }),
|
||||
},
|
||||
)]);
|
||||
let valid = rmp_serde::to_vec_named(&kwargs).unwrap();
|
||||
assert_eq!(decode_inline_mm_kwargs(&valid).unwrap(), kwargs);
|
||||
let mut trailing = valid.clone();
|
||||
trailing.push(0);
|
||||
let mut nested_kwargs = kwargs.clone();
|
||||
let mut value = MmKwargValue::Int(1);
|
||||
for _ in 0..64 {
|
||||
value = MmKwargValue::List(vec![value]);
|
||||
}
|
||||
nested_kwargs.get_mut("pixels").unwrap().data = Some(value);
|
||||
let nested = rmp_serde::to_vec_named(&nested_kwargs).unwrap();
|
||||
// A declared array length must not drive a proportional allocation.
|
||||
let huge_array = vec![
|
||||
0x81, 0xa1, b'x', 0x81, 0xa4, b'd', b'a', b't', b'a', 0xdd, 0xff, 0xff, 0xff, 0xff,
|
||||
];
|
||||
for bytes in [&valid[..valid.len() - 1], &trailing, &nested, &huge_array] {
|
||||
assert!(decode_inline_mm_kwargs(bytes).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kwargs_decode_python_encoder_tensor_and_factory_tuple() {
|
||||
// MsgpackEncoder(size_threshold=1 << 30), vLLM 6cbb3c154e:
|
||||
// pixel_values = tensor([1., 2.]), MultiModalFlatField(slices=[slice(0, 2)]).
|
||||
let bytes = b"\x81\xacpixel_values\x82\xa4data\x93\xa7float32\x91\x02\xd7\x03\
|
||||
\x00\x00\x80\x3f\x00\x00\x00\x40\
|
||||
\xa5field\x92\xa4flat\x83\xabkeep_on_cpu\xc2\
|
||||
\xa6slices\x91\x93\x00\x02\xc0\xa3dim\x00";
|
||||
let kwargs = decode_inline_mm_kwargs(bytes).unwrap();
|
||||
let expected = MmFieldElem {
|
||||
data: Some(MmKwargValue::Tensor(
|
||||
WireTensor::from_f32(vec![2], vec![1., 2.]).unwrap(),
|
||||
)),
|
||||
field: MmField::Flat(MmFlatField {
|
||||
slices: vec![MmSlice::Slice(SliceSpec {
|
||||
start: Some(0),
|
||||
stop: Some(2),
|
||||
step: None,
|
||||
})],
|
||||
dim: 0,
|
||||
keep_on_cpu: false,
|
||||
}),
|
||||
};
|
||||
assert_eq!(kwargs["pixel_values"], expected);
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ mod control;
|
||||
mod convert;
|
||||
mod health;
|
||||
mod inference;
|
||||
mod media;
|
||||
|
||||
/// Generated protobuf/gRPC types for the `vllm` package.
|
||||
pub use vllm_proto as pb;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::fs;
|
||||
use std::future::Future;
|
||||
use std::io;
|
||||
@@ -23,6 +24,7 @@ use tonic_health::pb::health_check_response::ServingStatus as HealthServingStatu
|
||||
use tonic_health::pb::health_client::HealthClient;
|
||||
use tonic_health::server::health_reporter;
|
||||
use tower::service_fn;
|
||||
use vllm_chat::multimodal::{MmLimitPerPrompt, MmLimitSpec, MmModality};
|
||||
use vllm_chat::{
|
||||
ChatBackend, ChatLlm, ChatRenderer, ChatRequest, ChatTextBackend, DefaultChatOutputProcessor,
|
||||
DynChatOutputProcessor, DynChatRenderer, NewChatOutputProcessorOptions, RenderedPrompt,
|
||||
@@ -32,12 +34,17 @@ use vllm_engine_core_client::mock_engine::{
|
||||
default_ready_response,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::decode_value;
|
||||
use vllm_engine_core_client::protocol::dtype::TensorDtype;
|
||||
use vllm_engine_core_client::protocol::handshake::{EngineCoreReadyResponse, KvEventsConfig};
|
||||
use vllm_engine_core_client::protocol::multimodal::{
|
||||
MmBatchedField, MmField, MmFieldElem, MmKwargValue,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::output::{
|
||||
EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, RequestBatchOutputs,
|
||||
UtilityCallOutput,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::request::EngineCoreRequest;
|
||||
use vllm_engine_core_client::protocol::tensor::WireTensor;
|
||||
use vllm_engine_core_client::protocol::utility::{UtilityOutput, UtilityResultEnvelope};
|
||||
use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task_with_ready};
|
||||
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId, TransportMode};
|
||||
@@ -292,6 +299,12 @@ const QWEN_IMAGE_TOKEN_ID: u32 = 151655;
|
||||
const TINY_PNG_DATA_URI: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=";
|
||||
|
||||
fn multimodal_backend() -> Arc<dyn ChatTextBackend> {
|
||||
multimodal_backend_with_limits(Default::default())
|
||||
}
|
||||
|
||||
fn multimodal_backend_with_limits(
|
||||
limit_mm_per_prompt: MmLimitPerPrompt,
|
||||
) -> Arc<dyn ChatTextBackend> {
|
||||
let config_path = std::env::temp_dir().join(format!(
|
||||
"vllm-grpc-qwen-config-{}.json",
|
||||
uuid::Uuid::new_v4()
|
||||
@@ -309,7 +322,7 @@ fn multimodal_backend() -> Arc<dyn ChatTextBackend> {
|
||||
..Default::default()
|
||||
},
|
||||
Arc::new(TestTokenizer::new().with_regular_token("<|image_pad|>", QWEN_IMAGE_TOKEN_ID)),
|
||||
Default::default(),
|
||||
limit_mm_per_prompt,
|
||||
)
|
||||
.expect("load multimodal info")
|
||||
.expect("qwen multimodal info is registered");
|
||||
@@ -877,6 +890,249 @@ async fn unary_generate_prepares_multimodal_input_for_engine_core() {
|
||||
server_task.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn generate_forwards_preprocessed_multimodal_features() {
|
||||
for stream in [false, true] {
|
||||
let kwargs = BTreeMap::from([(
|
||||
"pixel_values".to_string(),
|
||||
MmFieldElem {
|
||||
data: Some(MmKwargValue::Tensor(WireTensor::from_raw(
|
||||
TensorDtype::U8,
|
||||
vec![3],
|
||||
vec![1, 2, 3],
|
||||
))),
|
||||
field: MmField::Batched(MmBatchedField { keep_on_cpu: false }),
|
||||
},
|
||||
)]);
|
||||
let encoded_kwargs = rmp_serde::to_vec_named(&kwargs).expect("encode multimodal kwargs");
|
||||
let expected_kwargs = kwargs.clone();
|
||||
let (inference_service, control_service, engine_health, engine_task) =
|
||||
setup_grpc_service_with_backend(
|
||||
b"engine-grpc-preprocessed-multimodal",
|
||||
default_stream_output_specs(),
|
||||
multimodal_backend(),
|
||||
move |request| {
|
||||
assert_eq!(
|
||||
request.prompt_token_ids.as_deref(),
|
||||
Some(&[11, 99, 99, 12][..])
|
||||
);
|
||||
let features = request.mm_features.as_ref().expect("multimodal features");
|
||||
assert_eq!(features.len(), 1);
|
||||
let feature = &features[0];
|
||||
assert_eq!(feature.modality.as_str(), "image");
|
||||
assert_eq!(feature.identifier, "image-hash-a");
|
||||
assert_eq!(feature.mm_hash.as_deref(), Some("image-hash-a"));
|
||||
assert_eq!(feature.mm_position.offset, 1);
|
||||
assert_eq!(feature.mm_position.length, 2);
|
||||
assert_eq!(feature.data.as_ref(), Some(&expected_kwargs));
|
||||
assert_eq!(
|
||||
feature.mm_position.is_embed,
|
||||
Some(WireTensor::from_bool(vec![2], vec![true, false]).unwrap())
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let (channel, server_task) = start_grpc_test_server(
|
||||
inference_service,
|
||||
control_service,
|
||||
engine_health,
|
||||
tokio_util::sync::CancellationToken::new(),
|
||||
)
|
||||
.await;
|
||||
let mut client = InferenceClient::new(channel);
|
||||
|
||||
let request = pb::GenerateRequest {
|
||||
request_id: "test-preprocessed-multimodal".to_string(),
|
||||
model: "test-model".to_string(),
|
||||
prompt: Some(pb::generate_request::Prompt::TokenIds(pb::TokenIds {
|
||||
ids: vec![11, 99, 99, 12],
|
||||
})),
|
||||
media: vec![pb::MediaItem {
|
||||
modality: pb::Modality::Image as i32,
|
||||
source: Some(pb::media_item::Source::Features(
|
||||
pb::PreprocessedMediaFeatures {
|
||||
kwargs: Some(encoded_kwargs),
|
||||
identifier: "image-hash-a".to_string(),
|
||||
offset: 1,
|
||||
length: 2,
|
||||
mm_hash: Some("image-hash-a".to_string()),
|
||||
is_embed: vec![true, false],
|
||||
},
|
||||
)),
|
||||
mime_type: String::new(),
|
||||
uuid: String::new(),
|
||||
}],
|
||||
stopping: Some(pb::StoppingCriteria {
|
||||
max_new_tokens: 10,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
if stream {
|
||||
let mut response =
|
||||
client.generate_stream(request).await.expect("preprocessed stream").into_inner();
|
||||
while response.message().await.expect("stream message").is_some() {}
|
||||
} else {
|
||||
client.generate(request).await.expect("preprocessed generate");
|
||||
}
|
||||
|
||||
engine_task.await.expect("mock engine task");
|
||||
server_task.abort();
|
||||
}
|
||||
}
|
||||
|
||||
fn preprocessed_grpc_media(
|
||||
modality: pb::Modality,
|
||||
identifier: &str,
|
||||
offset: u64,
|
||||
length: u64,
|
||||
) -> pb::MediaItem {
|
||||
let kwargs = BTreeMap::from([(
|
||||
"pixel_values".to_string(),
|
||||
MmFieldElem {
|
||||
data: Some(MmKwargValue::Tensor(WireTensor::from_raw(
|
||||
TensorDtype::U8,
|
||||
vec![1],
|
||||
vec![1],
|
||||
))),
|
||||
field: MmField::Batched(MmBatchedField { keep_on_cpu: false }),
|
||||
},
|
||||
)]);
|
||||
pb::MediaItem {
|
||||
modality: modality as i32,
|
||||
source: Some(pb::media_item::Source::Features(
|
||||
pb::PreprocessedMediaFeatures {
|
||||
kwargs: Some(rmp_serde::to_vec_named(&kwargs).expect("encode kwargs")),
|
||||
identifier: identifier.to_string(),
|
||||
offset,
|
||||
length,
|
||||
mm_hash: Some(identifier.to_string()),
|
||||
is_embed: Vec::new(),
|
||||
},
|
||||
)),
|
||||
mime_type: String::new(),
|
||||
uuid: String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn preprocessed_grpc_request(media: Vec<pb::MediaItem>) -> pb::GenerateRequest {
|
||||
pb::GenerateRequest {
|
||||
request_id: "test-preprocessed-validation".to_string(),
|
||||
model: "test-model".to_string(),
|
||||
prompt: Some(pb::generate_request::Prompt::TokenIds(pb::TokenIds {
|
||||
ids: vec![11, 99, 99, 12],
|
||||
})),
|
||||
media,
|
||||
stopping: Some(pb::StoppingCriteria {
|
||||
max_new_tokens: 10,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn unary_generate_rejects_preprocessed_features_for_text_only_model() {
|
||||
let (inference_service, control_service, engine_health, _engine_task) =
|
||||
setup_grpc_service_with_backend(
|
||||
b"engine-grpc-preprocessed-text-only",
|
||||
default_stream_output_specs(),
|
||||
Arc::new(FakeTextBackend),
|
||||
|_| {},
|
||||
)
|
||||
.await;
|
||||
let (channel, server_task) = start_grpc_test_server(
|
||||
inference_service,
|
||||
control_service,
|
||||
engine_health,
|
||||
tokio_util::sync::CancellationToken::new(),
|
||||
)
|
||||
.await;
|
||||
let mut client = InferenceClient::new(channel);
|
||||
|
||||
let status = client
|
||||
.generate(preprocessed_grpc_request(vec![preprocessed_grpc_media(
|
||||
pb::Modality::Image,
|
||||
"image-a",
|
||||
1,
|
||||
2,
|
||||
)]))
|
||||
.await
|
||||
.expect_err("text-only models must reject preprocessed features");
|
||||
|
||||
assert_eq!(status.code(), tonic::Code::InvalidArgument);
|
||||
server_task.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn generate_validates_preprocessed_model_limits_and_prompt() {
|
||||
let limits = MmLimitPerPrompt::from([(MmModality::Image, MmLimitSpec::Count(1))]);
|
||||
let (inference_service, control_service, engine_health, _engine_task) =
|
||||
setup_grpc_service_with_backend(
|
||||
b"engine-grpc-preprocessed-limit",
|
||||
default_stream_output_specs(),
|
||||
multimodal_backend_with_limits(limits),
|
||||
|_| {},
|
||||
)
|
||||
.await;
|
||||
let (channel, server_task) = start_grpc_test_server(
|
||||
inference_service,
|
||||
control_service,
|
||||
engine_health,
|
||||
tokio_util::sync::CancellationToken::new(),
|
||||
)
|
||||
.await;
|
||||
let mut client = InferenceClient::new(channel);
|
||||
|
||||
let over_limit = preprocessed_grpc_request(vec![
|
||||
preprocessed_grpc_media(pb::Modality::Image, "image-a", 0, 1),
|
||||
preprocessed_grpc_media(pb::Modality::Image, "image-b", 2, 1),
|
||||
]);
|
||||
let bad_range = preprocessed_grpc_request(vec![preprocessed_grpc_media(
|
||||
pb::Modality::Image,
|
||||
"image",
|
||||
3,
|
||||
2,
|
||||
)]);
|
||||
let unsupported = preprocessed_grpc_request(vec![preprocessed_grpc_media(
|
||||
pb::Modality::Audio,
|
||||
"audio",
|
||||
0,
|
||||
1,
|
||||
)]);
|
||||
let mut text = preprocessed_grpc_request(vec![preprocessed_grpc_media(
|
||||
pb::Modality::Image,
|
||||
"image",
|
||||
0,
|
||||
1,
|
||||
)]);
|
||||
let mut truncated = text.clone();
|
||||
text.prompt = Some(pb::generate_request::Prompt::Text("hello".to_owned()));
|
||||
truncated.truncate_prompt_tokens = 1;
|
||||
let empty_identifier =
|
||||
preprocessed_grpc_request(vec![preprocessed_grpc_media(pb::Modality::Image, "", 0, 1)]);
|
||||
for request in [
|
||||
over_limit,
|
||||
bad_range,
|
||||
unsupported,
|
||||
text,
|
||||
truncated,
|
||||
empty_identifier,
|
||||
] {
|
||||
let status = client.generate(request.clone()).await.expect_err("invalid multimodal input");
|
||||
assert_eq!(status.code(), tonic::Code::InvalidArgument);
|
||||
let status = client
|
||||
.generate_stream(request)
|
||||
.await
|
||||
.expect_err("invalid streaming multimodal input");
|
||||
assert_eq!(status.code(), tonic::Code::InvalidArgument);
|
||||
}
|
||||
server_task.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn streaming_generate_rejects_text_prompt_with_media() {
|
||||
|
||||
Reference in New Issue
Block a user