[Rust Frontend][Multimodal] Accept preprocessed multimodal gRPC features (#55047)

Signed-off-by: Biswa Panda <biswa.panda@gmail.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Biswa Panda
2026-09-12 02:29:49 +00:00
committed by GitHub
co-authored by Bugen Zhao Codex
parent a0844fa6c6
commit 31759ccb6d
16 changed files with 986 additions and 113 deletions
+1 -1
View File
@@ -20,6 +20,7 @@ libc.workspace = true
llm-multimodal.workspace = true
openssl.workspace = true
prost-types.workspace = true
rmp-serde.workspace = true
rmpv.workspace = true
serde.workspace = true
serde_json.workspace = true
@@ -58,7 +59,6 @@ async-openai = { workspace = true, features = ["full"] }
bytes.workspace = true
clap.workspace = true
expect-test.workspace = true
rmp-serde.workspace = true
serial_test.workspace = true
tempfile.workspace = true
tokio = { workspace = true, features = ["test-util"] }
-91
View File
@@ -5,9 +5,7 @@
//! request/response types.
use tonic::Status;
use url::Url;
use uuid::Uuid;
use vllm_chat::MediaContentPart;
use vllm_engine_core_client::protocol::output::StopReason;
use vllm_engine_core_client::protocol::structured_outputs::StructuredOutputsParams;
use vllm_text::{
@@ -17,95 +15,6 @@ use vllm_text::{
use super::pb;
pub fn media_parts_from_request(
media: Vec<pb::MediaItem>,
) -> Result<Vec<MediaContentPart>, Status> {
let mut parts = Vec::with_capacity(media.len());
for (index, item) in media.into_iter().enumerate() {
let modality = item.modality();
if modality == pb::Modality::Unspecified {
return Err(Status::invalid_argument(format!(
"media[{index}].modality is required"
)));
}
let uuid = (!item.uuid.is_empty()).then_some(item.uuid);
let mime_type = (!item.mime_type.is_empty()).then_some(item.mime_type);
let source = item.source.ok_or_else(|| {
Status::invalid_argument(format!("media[{index}].source is required"))
})?;
match &source {
pb::media_item::Source::Url(url) => {
validate_media_uri(index, "url", url, &["http", "https"])?;
}
pb::media_item::Source::DataUri(uri) => {
validate_media_uri(index, "data_uri", uri, &["data"])?;
}
pb::media_item::Source::RawBytes(_) => {}
}
let part = match (modality, source) {
(
pb::Modality::Image,
pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url),
) => MediaContentPart::ImageUrl {
url,
detail: None,
uuid,
},
(pb::Modality::Image, pb::media_item::Source::RawBytes(data)) => {
MediaContentPart::ImageData {
data,
mime_type,
uuid,
detail: None,
}
}
(
pb::Modality::Video,
pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url),
) => MediaContentPart::VideoUrl { url, uuid },
(pb::Modality::Video, pb::media_item::Source::RawBytes(data)) => {
MediaContentPart::VideoData {
data,
mime_type,
uuid,
}
}
(
pb::Modality::Audio,
pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url),
) => MediaContentPart::AudioUrl { url, uuid },
(pb::Modality::Audio, pb::media_item::Source::RawBytes(data)) => {
MediaContentPart::AudioData {
data,
mime_type,
uuid,
}
}
(pb::Modality::Unspecified, _) => unreachable!("modality validated above"),
};
parts.push(part);
}
Ok(parts)
}
fn validate_media_uri(
index: usize,
field: &str,
value: &str,
allowed_schemes: &[&str],
) -> Result<(), Status> {
let uri = Url::parse(value).map_err(|_| {
Status::invalid_argument(format!("media[{index}].{field} is not a valid URI"))
})?;
if !allowed_schemes.contains(&uri.scheme()) {
return Err(Status::invalid_argument(format!(
"media[{index}].{field} must use the {} scheme",
allowed_schemes.join(" or ")
)));
}
Ok(())
}
// ========================================================================================
// Request conversion
// ========================================================================================
+11 -7
View File
@@ -149,19 +149,23 @@ impl InferenceServiceImpl {
})?);
}
let media = convert::media_parts_from_request(media)?;
let media = super::media::from_proto(media)?;
if !media.is_empty() {
let Prompt::TokenIds(mut token_ids) = text_request.prompt else {
return Err(Status::invalid_argument(
"multimodal gRPC requests must provide token_ids input",
));
};
let mm_features = self
.state
.chat
.prepare_media(media, &mut token_ids)
.await
.map_err(|error| Status::internal(error.to_report_string()))?;
let mm_features =
self.state.chat.prepare_media(media, &mut token_ids).await.map_err(
|error| {
if error.is_request_validation_error() {
Status::invalid_argument(error.to_report_string())
} else {
Status::internal(error.to_report_string())
}
},
)?;
text_request.prompt = Prompt::TokenIds(token_ids);
text_request.mm_features = mm_features;
}
+341
View File
@@ -0,0 +1,341 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//! Protobuf media adaptation into the shared multimodal preparation boundary.
use std::io::Cursor;
use serde::Deserialize as _;
use thiserror_ext::AsReport as _;
use tonic::Status;
use url::Url;
use vllm_chat::MediaContentPart;
use vllm_chat::multimodal::MultimodalInput;
use vllm_engine_core_client::protocol::multimodal::{
MmFeatureSpec, MmKwargsItem, MmModality, PlaceholderRange,
};
use vllm_engine_core_client::protocol::tensor::WireTensor;
use super::pb;
/// Maximum total serialized kwargs size accepted in one gRPC request.
const MAX_INLINE_MM_BYTES: usize = 16 * 1024 * 1024;
const MAX_MSGPACK_DEPTH: usize = 32;
/// Adapt a request containing either raw media or preprocessed features.
///
/// Require a single input kind and bound the total kwargs size before decoding.
/// Shared media preparation validates features, model support, item limits, and prompt bounds.
pub(super) fn from_proto(media: Vec<pb::MediaItem>) -> Result<MultimodalInput, Status> {
if !media
.iter()
.any(|item| matches!(item.source, Some(pb::media_item::Source::Features(_))))
{
return raw_parts_from_proto(media).map(MultimodalInput::Raw);
}
let mut encoded_bytes = 0usize;
for item in &media {
let Some(pb::media_item::Source::Features(feature)) = &item.source else {
return Err(mixed_media_error());
};
encoded_bytes = encoded_bytes.saturating_add(feature.kwargs.as_ref().map_or(0, Vec::len));
}
if encoded_bytes > MAX_INLINE_MM_BYTES {
return Err(Status::resource_exhausted(format!(
"inline multimodal payload exceeds {MAX_INLINE_MM_BYTES} bytes"
)));
}
let features = media
.into_iter()
.enumerate()
.map(|(index, item)| {
feature_from_proto(item).map_err(|error| {
Status::new(error.code(), format!("media[{index}]: {}", error.message()))
})
})
.collect::<Result<Vec<_>, _>>()?;
Ok(MultimodalInput::Preprocessed(features))
}
fn feature_from_proto(item: pb::MediaItem) -> Result<MmFeatureSpec, Status> {
let modality = match item.modality() {
pb::Modality::Image => MmModality::Image,
pb::Modality::Video => MmModality::Video,
pb::Modality::Audio => MmModality::Audio,
pb::Modality::Unspecified => return Err(Status::invalid_argument("modality is required")),
};
let Some(pb::media_item::Source::Features(feature)) = item.source else {
return Err(mixed_media_error());
};
let bytes = feature.kwargs.ok_or_else(|| {
Status::invalid_argument("features.kwargs is required; cache-only features are unsupported")
})?;
let offset = usize::try_from(feature.offset)
.map_err(|_| Status::invalid_argument("features.offset exceeds platform limits"))?;
let length = usize::try_from(feature.length)
.map_err(|_| Status::invalid_argument("features.length exceeds platform limits"))?;
let is_embed = if feature.is_embed.is_empty() {
None
} else {
Some(
WireTensor::from_bool(vec![length], feature.is_embed)
.map_err(Status::invalid_argument)?,
)
};
Ok(MmFeatureSpec {
modality,
data: Some(decode_inline_mm_kwargs(&bytes)?),
identifier: feature.identifier,
mm_hash: feature.mm_hash,
mm_position: PlaceholderRange {
offset,
length,
is_embed,
},
})
}
/// Decode one inline item using the engine protocol's typed serde schema.
fn decode_inline_mm_kwargs(bytes: &[u8]) -> Result<MmKwargsItem, Status> {
let mut decoder = rmp_serde::Deserializer::new(Cursor::new(bytes));
decoder.set_max_depth(MAX_MSGPACK_DEPTH);
let kwargs = MmKwargsItem::deserialize(&mut decoder).map_err(|error| {
Status::invalid_argument(format!(
"invalid multimodal kwargs MessagePack: {}",
error.as_report()
))
})?;
if decoder.position() != bytes.len() as u64 {
return Err(Status::invalid_argument(
"kwargs contains trailing MessagePack data",
));
}
Ok(kwargs)
}
fn mixed_media_error() -> Status {
Status::invalid_argument("raw media and preprocessed media features cannot be mixed")
}
fn raw_parts_from_proto(media: Vec<pb::MediaItem>) -> Result<Vec<MediaContentPart>, Status> {
let mut parts = Vec::with_capacity(media.len());
for (index, item) in media.into_iter().enumerate() {
let modality = item.modality();
if modality == pb::Modality::Unspecified {
return Err(Status::invalid_argument(format!(
"media[{index}].modality is required"
)));
}
let uuid = (!item.uuid.is_empty()).then_some(item.uuid);
let mime_type = (!item.mime_type.is_empty()).then_some(item.mime_type);
let source = item.source.ok_or_else(|| {
Status::invalid_argument(format!("media[{index}].source is required"))
})?;
match &source {
pb::media_item::Source::Url(url) => {
validate_media_uri(index, "url", url, &["http", "https"])?;
}
pb::media_item::Source::DataUri(uri) => {
validate_media_uri(index, "data_uri", uri, &["data"])?;
}
pb::media_item::Source::RawBytes(_) => {}
pb::media_item::Source::Features(_) => return Err(mixed_media_error()),
}
let part = match (modality, source) {
(
pb::Modality::Image,
pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url),
) => MediaContentPart::ImageUrl {
url,
detail: None,
uuid,
},
(pb::Modality::Image, pb::media_item::Source::RawBytes(data)) => {
MediaContentPart::ImageData {
data,
mime_type,
uuid,
detail: None,
}
}
(
pb::Modality::Video,
pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url),
) => MediaContentPart::VideoUrl { url, uuid },
(pb::Modality::Video, pb::media_item::Source::RawBytes(data)) => {
MediaContentPart::VideoData {
data,
mime_type,
uuid,
}
}
(
pb::Modality::Audio,
pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url),
) => MediaContentPart::AudioUrl { url, uuid },
(pb::Modality::Audio, pb::media_item::Source::RawBytes(data)) => {
MediaContentPart::AudioData {
data,
mime_type,
uuid,
}
}
(_, pb::media_item::Source::Features(_)) => return Err(mixed_media_error()),
(pb::Modality::Unspecified, _) => unreachable!("modality validated above"),
};
parts.push(part);
}
Ok(parts)
}
fn validate_media_uri(
index: usize,
field: &str,
value: &str,
allowed_schemes: &[&str],
) -> Result<(), Status> {
let uri = Url::parse(value).map_err(|_| {
Status::invalid_argument(format!("media[{index}].{field} is not a valid URI"))
})?;
if !allowed_schemes.contains(&uri.scheme()) {
return Err(Status::invalid_argument(format!(
"media[{index}].{field} must use the {} scheme",
allowed_schemes.join(" or ")
)));
}
Ok(())
}
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
use super::*;
use vllm_engine_core_client::protocol::multimodal::{
MmBatchedField, MmField, MmFieldElem, MmFlatField, MmKwargValue, MmSlice, SliceSpec,
};
fn preprocessed() -> pb::MediaItem {
let kwargs = BTreeMap::from([(
"x",
MmFieldElem {
data: Some(MmKwargValue::Int(1)),
field: MmField::Batched(MmBatchedField { keep_on_cpu: false }),
},
)]);
pb::MediaItem {
modality: pb::Modality::Image as i32,
source: Some(pb::media_item::Source::Features(
pb::PreprocessedMediaFeatures {
kwargs: Some(rmp_serde::to_vec_named(&kwargs).unwrap()),
identifier: "image-id".to_owned(),
length: 1,
..Default::default()
},
)),
..Default::default()
}
}
#[test]
fn mixed_media_is_rejected_in_both_orders() {
let raw = pb::MediaItem {
modality: pb::Modality::Image as i32,
source: Some(pb::media_item::Source::RawBytes(vec![1])),
..Default::default()
};
for media in [vec![raw.clone(), preprocessed()], vec![preprocessed(), raw]] {
assert_eq!(
from_proto(media).unwrap_err().code(),
tonic::Code::InvalidArgument
);
}
}
#[test]
fn protobuf_features_require_modality_inline_kwargs_and_a_matching_mask() {
let mut missing_modality = preprocessed();
missing_modality.modality = 99;
let mut missing_kwargs = preprocessed();
let Some(pb::media_item::Source::Features(feature)) = &mut missing_kwargs.source else {
unreachable!()
};
feature.kwargs = None;
let mut invalid_mask = preprocessed();
let Some(pb::media_item::Source::Features(feature)) = &mut invalid_mask.source else {
unreachable!()
};
feature.is_embed = vec![true, false];
for media in [missing_modality, missing_kwargs, invalid_mask] {
let error = from_proto(vec![media]).unwrap_err();
assert_eq!(error.code(), tonic::Code::InvalidArgument);
assert!(error.message().starts_with("media[0]:"));
}
}
#[test]
fn aggregate_payload_limit_is_checked_before_decoding_items() {
let mut item = preprocessed();
let Some(pb::media_item::Source::Features(feature)) = &mut item.source else {
unreachable!()
};
feature.kwargs = Some(vec![0; MAX_INLINE_MM_BYTES / 2 + 1]);
let error = from_proto(vec![item.clone(), item]).unwrap_err();
assert_eq!(error.code(), tonic::Code::ResourceExhausted);
}
#[test]
fn kwargs_decoder_rejects_trailing_truncated_and_deeply_nested_input() {
let kwargs = BTreeMap::from([(
"pixels".to_owned(),
MmFieldElem {
data: Some(MmKwargValue::Int(1)),
field: MmField::Batched(MmBatchedField { keep_on_cpu: false }),
},
)]);
let valid = rmp_serde::to_vec_named(&kwargs).unwrap();
assert_eq!(decode_inline_mm_kwargs(&valid).unwrap(), kwargs);
let mut trailing = valid.clone();
trailing.push(0);
let mut nested_kwargs = kwargs.clone();
let mut value = MmKwargValue::Int(1);
for _ in 0..64 {
value = MmKwargValue::List(vec![value]);
}
nested_kwargs.get_mut("pixels").unwrap().data = Some(value);
let nested = rmp_serde::to_vec_named(&nested_kwargs).unwrap();
// A declared array length must not drive a proportional allocation.
let huge_array = vec![
0x81, 0xa1, b'x', 0x81, 0xa4, b'd', b'a', b't', b'a', 0xdd, 0xff, 0xff, 0xff, 0xff,
];
for bytes in [&valid[..valid.len() - 1], &trailing, &nested, &huge_array] {
assert!(decode_inline_mm_kwargs(bytes).is_err());
}
}
#[test]
fn kwargs_decode_python_encoder_tensor_and_factory_tuple() {
// MsgpackEncoder(size_threshold=1 << 30), vLLM 6cbb3c154e:
// pixel_values = tensor([1., 2.]), MultiModalFlatField(slices=[slice(0, 2)]).
let bytes = b"\x81\xacpixel_values\x82\xa4data\x93\xa7float32\x91\x02\xd7\x03\
\x00\x00\x80\x3f\x00\x00\x00\x40\
\xa5field\x92\xa4flat\x83\xabkeep_on_cpu\xc2\
\xa6slices\x91\x93\x00\x02\xc0\xa3dim\x00";
let kwargs = decode_inline_mm_kwargs(bytes).unwrap();
let expected = MmFieldElem {
data: Some(MmKwargValue::Tensor(
WireTensor::from_f32(vec![2], vec![1., 2.]).unwrap(),
)),
field: MmField::Flat(MmFlatField {
slices: vec![MmSlice::Slice(SliceSpec {
start: Some(0),
stop: Some(2),
step: None,
})],
dim: 0,
keep_on_cpu: false,
}),
};
assert_eq!(kwargs["pixel_values"], expected);
}
}
+1
View File
@@ -7,6 +7,7 @@ mod control;
mod convert;
mod health;
mod inference;
mod media;
/// Generated protobuf/gRPC types for the `vllm` package.
pub use vllm_proto as pb;
+257 -1
View File
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::collections::BTreeMap;
use std::fs;
use std::future::Future;
use std::io;
@@ -23,6 +24,7 @@ use tonic_health::pb::health_check_response::ServingStatus as HealthServingStatu
use tonic_health::pb::health_client::HealthClient;
use tonic_health::server::health_reporter;
use tower::service_fn;
use vllm_chat::multimodal::{MmLimitPerPrompt, MmLimitSpec, MmModality};
use vllm_chat::{
ChatBackend, ChatLlm, ChatRenderer, ChatRequest, ChatTextBackend, DefaultChatOutputProcessor,
DynChatOutputProcessor, DynChatRenderer, NewChatOutputProcessorOptions, RenderedPrompt,
@@ -32,12 +34,17 @@ use vllm_engine_core_client::mock_engine::{
default_ready_response,
};
use vllm_engine_core_client::protocol::decode_value;
use vllm_engine_core_client::protocol::dtype::TensorDtype;
use vllm_engine_core_client::protocol::handshake::{EngineCoreReadyResponse, KvEventsConfig};
use vllm_engine_core_client::protocol::multimodal::{
MmBatchedField, MmField, MmFieldElem, MmKwargValue,
};
use vllm_engine_core_client::protocol::output::{
EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, RequestBatchOutputs,
UtilityCallOutput,
};
use vllm_engine_core_client::protocol::request::EngineCoreRequest;
use vllm_engine_core_client::protocol::tensor::WireTensor;
use vllm_engine_core_client::protocol::utility::{UtilityOutput, UtilityResultEnvelope};
use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task_with_ready};
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId, TransportMode};
@@ -292,6 +299,12 @@ const QWEN_IMAGE_TOKEN_ID: u32 = 151655;
const TINY_PNG_DATA_URI: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=";
fn multimodal_backend() -> Arc<dyn ChatTextBackend> {
multimodal_backend_with_limits(Default::default())
}
fn multimodal_backend_with_limits(
limit_mm_per_prompt: MmLimitPerPrompt,
) -> Arc<dyn ChatTextBackend> {
let config_path = std::env::temp_dir().join(format!(
"vllm-grpc-qwen-config-{}.json",
uuid::Uuid::new_v4()
@@ -309,7 +322,7 @@ fn multimodal_backend() -> Arc<dyn ChatTextBackend> {
..Default::default()
},
Arc::new(TestTokenizer::new().with_regular_token("<|image_pad|>", QWEN_IMAGE_TOKEN_ID)),
Default::default(),
limit_mm_per_prompt,
)
.expect("load multimodal info")
.expect("qwen multimodal info is registered");
@@ -877,6 +890,249 @@ async fn unary_generate_prepares_multimodal_input_for_engine_core() {
server_task.abort();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn generate_forwards_preprocessed_multimodal_features() {
for stream in [false, true] {
let kwargs = BTreeMap::from([(
"pixel_values".to_string(),
MmFieldElem {
data: Some(MmKwargValue::Tensor(WireTensor::from_raw(
TensorDtype::U8,
vec![3],
vec![1, 2, 3],
))),
field: MmField::Batched(MmBatchedField { keep_on_cpu: false }),
},
)]);
let encoded_kwargs = rmp_serde::to_vec_named(&kwargs).expect("encode multimodal kwargs");
let expected_kwargs = kwargs.clone();
let (inference_service, control_service, engine_health, engine_task) =
setup_grpc_service_with_backend(
b"engine-grpc-preprocessed-multimodal",
default_stream_output_specs(),
multimodal_backend(),
move |request| {
assert_eq!(
request.prompt_token_ids.as_deref(),
Some(&[11, 99, 99, 12][..])
);
let features = request.mm_features.as_ref().expect("multimodal features");
assert_eq!(features.len(), 1);
let feature = &features[0];
assert_eq!(feature.modality.as_str(), "image");
assert_eq!(feature.identifier, "image-hash-a");
assert_eq!(feature.mm_hash.as_deref(), Some("image-hash-a"));
assert_eq!(feature.mm_position.offset, 1);
assert_eq!(feature.mm_position.length, 2);
assert_eq!(feature.data.as_ref(), Some(&expected_kwargs));
assert_eq!(
feature.mm_position.is_embed,
Some(WireTensor::from_bool(vec![2], vec![true, false]).unwrap())
);
},
)
.await;
let (channel, server_task) = start_grpc_test_server(
inference_service,
control_service,
engine_health,
tokio_util::sync::CancellationToken::new(),
)
.await;
let mut client = InferenceClient::new(channel);
let request = pb::GenerateRequest {
request_id: "test-preprocessed-multimodal".to_string(),
model: "test-model".to_string(),
prompt: Some(pb::generate_request::Prompt::TokenIds(pb::TokenIds {
ids: vec![11, 99, 99, 12],
})),
media: vec![pb::MediaItem {
modality: pb::Modality::Image as i32,
source: Some(pb::media_item::Source::Features(
pb::PreprocessedMediaFeatures {
kwargs: Some(encoded_kwargs),
identifier: "image-hash-a".to_string(),
offset: 1,
length: 2,
mm_hash: Some("image-hash-a".to_string()),
is_embed: vec![true, false],
},
)),
mime_type: String::new(),
uuid: String::new(),
}],
stopping: Some(pb::StoppingCriteria {
max_new_tokens: 10,
..Default::default()
}),
..Default::default()
};
if stream {
let mut response =
client.generate_stream(request).await.expect("preprocessed stream").into_inner();
while response.message().await.expect("stream message").is_some() {}
} else {
client.generate(request).await.expect("preprocessed generate");
}
engine_task.await.expect("mock engine task");
server_task.abort();
}
}
fn preprocessed_grpc_media(
modality: pb::Modality,
identifier: &str,
offset: u64,
length: u64,
) -> pb::MediaItem {
let kwargs = BTreeMap::from([(
"pixel_values".to_string(),
MmFieldElem {
data: Some(MmKwargValue::Tensor(WireTensor::from_raw(
TensorDtype::U8,
vec![1],
vec![1],
))),
field: MmField::Batched(MmBatchedField { keep_on_cpu: false }),
},
)]);
pb::MediaItem {
modality: modality as i32,
source: Some(pb::media_item::Source::Features(
pb::PreprocessedMediaFeatures {
kwargs: Some(rmp_serde::to_vec_named(&kwargs).expect("encode kwargs")),
identifier: identifier.to_string(),
offset,
length,
mm_hash: Some(identifier.to_string()),
is_embed: Vec::new(),
},
)),
mime_type: String::new(),
uuid: String::new(),
}
}
fn preprocessed_grpc_request(media: Vec<pb::MediaItem>) -> pb::GenerateRequest {
pb::GenerateRequest {
request_id: "test-preprocessed-validation".to_string(),
model: "test-model".to_string(),
prompt: Some(pb::generate_request::Prompt::TokenIds(pb::TokenIds {
ids: vec![11, 99, 99, 12],
})),
media,
stopping: Some(pb::StoppingCriteria {
max_new_tokens: 10,
..Default::default()
}),
..Default::default()
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn unary_generate_rejects_preprocessed_features_for_text_only_model() {
let (inference_service, control_service, engine_health, _engine_task) =
setup_grpc_service_with_backend(
b"engine-grpc-preprocessed-text-only",
default_stream_output_specs(),
Arc::new(FakeTextBackend),
|_| {},
)
.await;
let (channel, server_task) = start_grpc_test_server(
inference_service,
control_service,
engine_health,
tokio_util::sync::CancellationToken::new(),
)
.await;
let mut client = InferenceClient::new(channel);
let status = client
.generate(preprocessed_grpc_request(vec![preprocessed_grpc_media(
pb::Modality::Image,
"image-a",
1,
2,
)]))
.await
.expect_err("text-only models must reject preprocessed features");
assert_eq!(status.code(), tonic::Code::InvalidArgument);
server_task.abort();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn generate_validates_preprocessed_model_limits_and_prompt() {
let limits = MmLimitPerPrompt::from([(MmModality::Image, MmLimitSpec::Count(1))]);
let (inference_service, control_service, engine_health, _engine_task) =
setup_grpc_service_with_backend(
b"engine-grpc-preprocessed-limit",
default_stream_output_specs(),
multimodal_backend_with_limits(limits),
|_| {},
)
.await;
let (channel, server_task) = start_grpc_test_server(
inference_service,
control_service,
engine_health,
tokio_util::sync::CancellationToken::new(),
)
.await;
let mut client = InferenceClient::new(channel);
let over_limit = preprocessed_grpc_request(vec![
preprocessed_grpc_media(pb::Modality::Image, "image-a", 0, 1),
preprocessed_grpc_media(pb::Modality::Image, "image-b", 2, 1),
]);
let bad_range = preprocessed_grpc_request(vec![preprocessed_grpc_media(
pb::Modality::Image,
"image",
3,
2,
)]);
let unsupported = preprocessed_grpc_request(vec![preprocessed_grpc_media(
pb::Modality::Audio,
"audio",
0,
1,
)]);
let mut text = preprocessed_grpc_request(vec![preprocessed_grpc_media(
pb::Modality::Image,
"image",
0,
1,
)]);
let mut truncated = text.clone();
text.prompt = Some(pb::generate_request::Prompt::Text("hello".to_owned()));
truncated.truncate_prompt_tokens = 1;
let empty_identifier =
preprocessed_grpc_request(vec![preprocessed_grpc_media(pb::Modality::Image, "", 0, 1)]);
for request in [
over_limit,
bad_range,
unsupported,
text,
truncated,
empty_identifier,
] {
let status = client.generate(request.clone()).await.expect_err("invalid multimodal input");
assert_eq!(status.code(), tonic::Code::InvalidArgument);
let status = client
.generate_stream(request)
.await
.expect_err("invalid streaming multimodal input");
assert_eq!(status.code(), tonic::Code::InvalidArgument);
}
server_task.abort();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial]
async fn streaming_generate_rejects_text_prompt_with_media() {