From 41cffa9af0b7317d71838262004186b68e976773 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Thu, 10 Sep 2026 12:11:22 +1000 Subject: [PATCH] [Rust Frontend] Strongly type wire dtypes and multimodal modalities (#56174) Co-authored-by: Codex Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Bugen Zhao --- rust/src/chat/src/multimodal.rs | 61 ++--- rust/src/chat/src/multimodal/audio.rs | 8 +- rust/src/chat/src/multimodal/expand.rs | 2 +- rust/src/chat/src/multimodal/tensor.rs | 161 ++++-------- .../engine-core-client/src/protocol/dtype.rs | 232 +++++++++++++++++- .../src/protocol/logprobs.rs | 7 +- .../src/protocol/logprobs/array.rs | 72 ++---- .../src/protocol/logprobs/tests.rs | 23 ++ .../src/protocol/multimodal.rs | 22 +- .../src/protocol/request.rs | 12 +- .../engine-core-client/src/protocol/tensor.rs | 105 +++++--- .../engine-core-client/src/tests/client.rs | 6 +- rust/src/server/src/grpc/tests.rs | 2 +- rust/src/server/src/routes/tests.rs | 4 +- rust/src/text/src/lower.rs | 6 +- 15 files changed, 453 insertions(+), 270 deletions(-) diff --git a/rust/src/chat/src/multimodal.rs b/rust/src/chat/src/multimodal.rs index edb4722d48..174a0d4bb0 100644 --- a/rust/src/chat/src/multimodal.rs +++ b/rust/src/chat/src/multimodal.rs @@ -60,30 +60,9 @@ pub struct MultimodalModelInfo { /// Per-modality item-count limits configured by `--limit-mm-per-prompt`. /// /// Modalities absent from the map are unlimited. -pub type MmLimitPerPrompt = HashMap; +pub type MmLimitPerPrompt = HashMap; -/// Modalities that `--limit-mm-per-prompt` can be keyed by. -/// -/// Closed on purpose: these are exactly the keys Python accepts, per -/// `MultiModalDummyOptionsBuiltins` in `vllm/config/multimodal.py`. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum MmLimitModality { - Image, - Audio, - Video, -} - -impl MmLimitModality { - /// The wire name, matching Python's modality strings. - pub fn as_str(self) -> &'static str { - match self { - Self::Image => "image", - Self::Audio => "audio", - Self::Video => "video", - } - } -} +pub use vllm_engine_core_client::protocol::multimodal::MmModality; /// One modality's limit, in either of the two shapes Python accepts. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -646,17 +625,17 @@ fn input_audio_data_url(data: &str, format: Option<&str>) -> Result { /// Embedding inputs share their base modality's budget rather than getting one /// of their own, matching Python's `modality.replace("_embeds", "")` in /// `vllm/entrypoints/chat_utils.py`. -fn media_part_limit_modality(part: &MediaContentPart) -> Option { +fn media_part_limit_modality(part: &MediaContentPart) -> Option { match part { MediaContentPart::Text { .. } => None, MediaContentPart::ImageUrl { .. } | MediaContentPart::ImageData { .. } - | MediaContentPart::ImageEmbeds { .. } => Some(MmLimitModality::Image), + | MediaContentPart::ImageEmbeds { .. } => Some(MmModality::Image), MediaContentPart::AudioUrl { .. } | MediaContentPart::AudioData { .. } => { - Some(MmLimitModality::Audio) + Some(MmModality::Audio) } MediaContentPart::VideoUrl { .. } | MediaContentPart::VideoData { .. } => { - Some(MmLimitModality::Video) + Some(MmModality::Video) } } } @@ -667,7 +646,7 @@ impl MultimodalModelInfo { /// /// Modalities without a configured count are unlimited. fn validate_mm_limits(&self, media_parts: &[MediaContentPart]) -> Result<()> { - let mut counts: HashMap = HashMap::new(); + let mut counts: HashMap = HashMap::new(); for part in media_parts { if let Some(modality) = media_part_limit_modality(part) { *counts.entry(modality).or_default() += 1; @@ -738,7 +717,11 @@ impl MultimodalModelInfo { for (item, range) in izip!(media.items, media_ranges) { features.push(MmFeatureSpec { data: Some(item.data), - modality: media.modality.to_string(), + modality: match media.modality { + Modality::Image | Modality::ImageEmbeds => MmModality::Image, + Modality::Video => MmModality::Video, + Modality::Audio => MmModality::Audio, + }, identifier: item.uuid.unwrap_or_else(|| item.hash.clone()), mm_position: range, mm_hash: Some(item.hash), @@ -1029,10 +1012,8 @@ mod tests { #[test] fn validate_mm_limits_enforces_configured_limit_at_the_boundary() { - let info = qwen3_vl_info_with_limits(HashMap::from([( - MmLimitModality::Image, - MmLimitSpec::Count(1), - )])); + let info = + qwen3_vl_info_with_limits(HashMap::from([(MmModality::Image, MmLimitSpec::Count(1))])); assert!(info.validate_mm_limits(&[image_url_part()]).is_ok()); let error = info.validate_mm_limits(&[image_url_part(), image_url_part()]).unwrap_err(); @@ -1047,10 +1028,8 @@ mod tests { #[test] fn validate_mm_limits_counts_image_embeds_against_the_image_limit() { - let info = qwen3_vl_info_with_limits(HashMap::from([( - MmLimitModality::Image, - MmLimitSpec::Count(1), - )])); + let info = + qwen3_vl_info_with_limits(HashMap::from([(MmModality::Image, MmLimitSpec::Count(1))])); let image_embeds_part = MediaContentPart::ImageEmbeds { payload: serde_json::Value::String("AAAA".to_string()), uuid: None, @@ -1068,7 +1047,7 @@ mod tests { #[test] fn validate_mm_limits_treats_a_count_less_options_object_as_unlimited() { let info = qwen3_vl_info_with_limits(HashMap::from([( - MmLimitModality::Image, + MmModality::Image, MmLimitSpec::Options { count: None, extra: BTreeMap::from([("width".to_string(), serde_json::json!(512))]), @@ -1086,9 +1065,9 @@ mod tests { fn limit_map_parses_both_shapes_python_accepts() { let limits = parse_limits(r#"{"image": 16, "video": {"count": 1, "num_frames": 32}}"#); - assert_eq!(limits[&MmLimitModality::Image].count(), Some(16)); - assert_eq!(limits[&MmLimitModality::Video].count(), Some(1)); - assert_eq!(limits.get(&MmLimitModality::Audio), None); + assert_eq!(limits[&MmModality::Image].count(), Some(16)); + assert_eq!(limits[&MmModality::Video].count(), Some(1)); + assert_eq!(limits.get(&MmModality::Audio), None); } #[test] diff --git a/rust/src/chat/src/multimodal/audio.rs b/rust/src/chat/src/multimodal/audio.rs index 41ea47b484..897f7aa492 100644 --- a/rust/src/chat/src/multimodal/audio.rs +++ b/rust/src/chat/src/multimodal/audio.rs @@ -268,14 +268,14 @@ mod tests { assert!(matches!( features.data.as_ref(), Some(MmKwargValue::Tensor(tensor)) - if tensor.dtype == "float32" && tensor.shape.first() == Some(&128) + if tensor.dtype.as_str() == "float32" && tensor.shape.first() == Some(&128) )); let lengths = &item.data["audio_feature_lengths"]; assert!(matches!(&lengths.field, MmField::Batched(_))); assert!(matches!( lengths.data.as_ref(), Some(MmKwargValue::Tensor(tensor)) - if tensor.dtype == "int64" && tensor.shape.is_empty() + if tensor.dtype.as_str() == "int64" && tensor.shape.is_empty() )); } @@ -314,14 +314,14 @@ mod tests { assert!(matches!( features.data.as_ref(), Some(MmKwargValue::Tensor(tensor)) - if tensor.dtype == "float32" && tensor.shape.get(1) == Some(&80) + if tensor.dtype.as_str() == "float32" && tensor.shape.get(1) == Some(&80) )); let count = &item.data["num_audio_tokens"]; assert!(matches!(&count.field, MmField::Batched(_))); assert!(matches!( count.data.as_ref(), Some(MmKwargValue::Tensor(tensor)) - if tensor.dtype == "int64" && tensor.shape.is_empty() + if tensor.dtype.as_str() == "int64" && tensor.shape.is_empty() )); } } diff --git a/rust/src/chat/src/multimodal/expand.rs b/rust/src/chat/src/multimodal/expand.rs index 884df2fe44..d9c9c54b34 100644 --- a/rust/src/chat/src/multimodal/expand.rs +++ b/rust/src/chat/src/multimodal/expand.rs @@ -228,7 +228,7 @@ mod tests { fn assert_bool_mask(range: &PlaceholderRange, expected: &[bool]) { let tensor = range.is_embed.as_ref().expect("is_embed mask"); - assert_eq!(tensor.dtype, "bool"); + assert_eq!(tensor.dtype.as_str(), "bool"); assert_eq!(tensor.shape, vec![expected.len()]); assert_eq!( tensor.data, diff --git a/rust/src/chat/src/multimodal/tensor.rs b/rust/src/chat/src/multimodal/tensor.rs index d14d4c2669..2c564fbda7 100644 --- a/rust/src/chat/src/multimodal/tensor.rs +++ b/rust/src/chat/src/multimodal/tensor.rs @@ -7,54 +7,17 @@ use std::mem::size_of; use half::{bf16, f16}; use llm_multimodal::{ModelSpecificValue, PreprocessedEncoderInputs}; use ndarray::ArrayD; -use vllm_engine_core_client::protocol::dtype::ModelDtype; +use vllm_engine_core_client::protocol::dtype::{ModelDtype, TensorDtype}; use vllm_engine_core_client::protocol::multimodal::MmKwargValue as ProtocolKwargValue; -use vllm_engine_core_client::protocol::tensor::{ShapeExt as _, WireArrayData, WireTensor}; +use vllm_engine_core_client::protocol::tensor::{WireArrayData, WireTensor}; use crate::error::{Error, Result, bail_multimodal, multimodal}; -/// Element type retained alongside an encoded tensor during multimodal lowering. -#[derive(Debug, Clone, Copy)] -pub(super) enum TensorKind { - /// 32-bit floating point. - F32, - /// IEEE 16-bit floating point. - F16, - /// Brain floating point. - Bf16, - /// Signed 64-bit integer. - I64, - /// Unsigned 32-bit integer. - U32, -} - -impl TensorKind { - const fn element_size(self) -> usize { - match self { - Self::F32 => size_of::(), - Self::F16 => size_of::(), - Self::Bf16 => size_of::(), - Self::I64 => size_of::(), - Self::U32 => size_of::(), - } - } - - const fn wire_dtype(self) -> &'static str { - match self { - Self::F32 => "float32", - Self::F16 => "float16", - Self::Bf16 => "bfloat16", - Self::I64 => "int64", - Self::U32 => "uint32", - } - } -} - /// Representation for multimodal kwarg values for transformation. #[derive(Debug)] pub(super) enum KwargValue { /// Tensor with row-major flat data and shape. - Tensor { kind: TensorKind, wire: WireTensor }, + Tensor { wire: WireTensor }, /// Non-tensor kwarg value that is shared or copied as-is. Passthrough(ProtocolKwargValue), } @@ -96,11 +59,11 @@ impl KwargValue { } ModelSpecificValue::IntTensor { data, shape } => { let wire = WireTensor::from_i64(shape, data).map_err(Error::Multimodal)?; - Self::tensor(TensorKind::I64, wire) + Self::Tensor { wire } } ModelSpecificValue::UintTensor { data, shape } => { let wire = WireTensor::from_u32(shape, data).map_err(Error::Multimodal)?; - Self::tensor(TensorKind::U32, wire) + Self::Tensor { wire } } ModelSpecificValue::Int(value) => Self::Passthrough(Int(value)), ModelSpecificValue::Float(value) => Self::Passthrough(Float(value)), @@ -148,23 +111,16 @@ impl KwargValue { /// Convert a float tensor to the target float dtype if needed, keeping the /// same shape. fn from_f32_parts(data: Vec, shape: Vec, float_dtype: ModelDtype) -> Result { - let (kind, wire) = match float_dtype { - ModelDtype::Float16 => ( - TensorKind::F16, - WireTensor::from_f16(shape, data.into_iter().map(f16::from_f32).collect()), - ), - ModelDtype::BFloat16 => ( - TensorKind::Bf16, - WireTensor::from_bf16(shape, data.into_iter().map(bf16::from_f32).collect()), - ), - ModelDtype::Float32 => (TensorKind::F32, WireTensor::from_f32(shape, data)), + let wire = match float_dtype { + ModelDtype::Float16 => { + WireTensor::from_f16(shape, data.into_iter().map(f16::from_f32).collect()) + } + ModelDtype::BFloat16 => { + WireTensor::from_bf16(shape, data.into_iter().map(bf16::from_f32).collect()) + } + ModelDtype::Float32 => WireTensor::from_f32(shape, data), }; - wire.map(|wire| Self::tensor(kind, wire)).map_err(Error::Multimodal) - } - - fn tensor(kind: TensorKind, wire: WireTensor) -> Self { - debug_assert_eq!(wire.dtype, kind.wire_dtype()); - Self::Tensor { kind, wire } + wire.map(|wire| Self::Tensor { wire }).map_err(Error::Multimodal) } } @@ -215,9 +171,7 @@ impl KwargValue { drop_axis: bool, ) -> Result { let wire = match self { - Self::Tensor { kind, wire } => { - slice_first_axis_range(wire, kind.element_size(), start, end, drop_axis) - } + Self::Tensor { wire } => slice_first_axis_range(wire, start, end, drop_axis), Self::Passthrough(value) => return Ok(value.clone()), }; wire.map(ProtocolKwargValue::Tensor) @@ -244,10 +198,7 @@ pub(super) fn flat_range_for_index( /// Read a tensor value as per-image sizes for flat slicing. fn tensor_as_usize_vec(tensor: &KwargValue) -> Result> { match tensor { - KwargValue::Tensor { - kind: TensorKind::I64, - wire, - } => raw_tensor_bytes(wire, size_of::())? + KwargValue::Tensor { wire } if wire.dtype == TensorDtype::I64 => raw_tensor_bytes(wire)? .chunks_exact(size_of::()) .map(|bytes| i64::from_ne_bytes(bytes.try_into().expect("exact int64 chunk"))) .map(|value| { @@ -255,13 +206,14 @@ fn tensor_as_usize_vec(tensor: &KwargValue) -> Result> { .map_err(|_| multimodal!("negative flat tensor size `{value}`")) }) .collect(), - KwargValue::Tensor { - kind: TensorKind::U32, - wire, - } => Ok(raw_tensor_bytes(wire, size_of::())? - .chunks_exact(size_of::()) - .map(|bytes| u32::from_ne_bytes(bytes.try_into().expect("exact uint32 chunk")) as usize) - .collect()), + KwargValue::Tensor { wire } if wire.dtype == TensorDtype::U32 => { + Ok(raw_tensor_bytes(wire)? + .chunks_exact(size_of::()) + .map(|bytes| { + u32::from_ne_bytes(bytes.try_into().expect("exact uint32 chunk")) as usize + }) + .collect()) + } _ => Err(multimodal!("flat tensor sizes must be int64 or uint32")), } } @@ -269,13 +221,13 @@ fn tensor_as_usize_vec(tensor: &KwargValue) -> Result> { /// Slice a flat row-major tensor along its first axis. fn slice_first_axis_range( tensor: &WireTensor, - element_size: usize, start: usize, end: usize, drop_axis: bool, ) -> Result { let shape = tensor.shape.as_slice(); - raw_tensor_bytes(tensor, element_size)?; + raw_tensor_bytes(tensor)?; + let element_size = tensor.dtype.element_size(); let first_dim = *shape.first().ok_or_else(|| multimodal!("tensor has no first dimension"))?; if start > end || end > first_dim { bail_multimodal!("invalid tensor slice {start}..{end} for first dimension {first_dim}"); @@ -302,34 +254,15 @@ fn slice_first_axis_range( return Err(multimodal!("cannot slice an aux tensor buffer")); }; Ok(WireTensor::from_raw_bytes( - tensor.dtype.clone(), + tensor.dtype, out_shape, data.slice(data_start..data_end), )) } -fn raw_tensor_bytes(tensor: &WireTensor, element_size: usize) -> Result<&[u8]> { - let WireArrayData::RawView(data) = &tensor.data else { - return Err(multimodal!("expected an inline tensor buffer")); - }; - let expected_bytes = tensor - .shape - .checked_numel() - .and_then(|numel| numel.checked_mul(element_size)) - .ok_or_else(|| { - multimodal!( - "tensor shape {:?} byte length overflowed usize", - tensor.shape - ) - })?; - if expected_bytes != data.len() { - bail_multimodal!( - "tensor shape {:?} expects {expected_bytes} bytes, got {}", - tensor.shape, - data.len() - ); - } - Ok(data) +fn raw_tensor_bytes(tensor: &WireTensor) -> Result<&[u8]> { + tensor.validate_inline().map_err(Error::Multimodal)?; + Ok(tensor.data.as_raw_view().expect("validated inline tensor")) } #[cfg(test)] @@ -373,7 +306,7 @@ mod tests { }; assert_eq!(tensor.shape, shape, "{name} layout shape"); - assert_eq!(tensor.dtype, "float32", "{name} layout dtype"); + assert_eq!(tensor.dtype.as_str(), "float32", "{name} layout dtype"); assert_eq!( tensor.data.into_raw_view().unwrap(), expected, @@ -386,10 +319,9 @@ mod tests { fn batched_wire_value_at_drops_first_axis() { let data = vec![1.0_f32, 2.0, 3.0, 4.0]; let expected_ptr = data.as_ptr().cast::().wrapping_add(2 * size_of::()); - let value = KwargValue::tensor( - TensorKind::F32, - WireTensor::from_f32(vec![2, 2], data).unwrap(), - ); + let value = KwargValue::Tensor { + wire: WireTensor::from_f32(vec![2, 2], data).unwrap(), + }; let ProtocolKwargValue::Tensor(tensor) = value.batched_wire_value_at(1).unwrap() else { panic!("expected tensor"); @@ -406,10 +338,9 @@ mod tests { #[test] fn flat_wire_value_range_keeps_first_axis() { - let value = KwargValue::tensor( - TensorKind::U32, - WireTensor::from_u32(vec![5, 2], (0..10_u32).collect()).unwrap(), - ); + let value = KwargValue::Tensor { + wire: WireTensor::from_u32(vec![5, 2], (0..10_u32).collect()).unwrap(), + }; let ProtocolKwargValue::Tensor(tensor) = value.flat_wire_value_range(1, 3).unwrap() else { panic!("expected tensor"); @@ -424,10 +355,9 @@ mod tests { #[test] fn flat_range_for_index_uses_size_tensor() { - let sizes = KwargValue::tensor( - TensorKind::I64, - WireTensor::from_i64(vec![3], vec![2_i64, 3, 4]).unwrap(), - ); + let sizes = KwargValue::Tensor { + wire: WireTensor::from_i64(vec![3], vec![2_i64, 3, 4]).unwrap(), + }; assert_eq!( flat_range_for_index(&sizes, "image_grid_thw", 1).unwrap(), @@ -437,11 +367,12 @@ mod tests { #[test] fn slice_first_axis_range_errors_on_shape_data_mismatch() { - let tensor = WireTensor::from_raw("float32", vec![2, 2], vec![0; 3 * size_of::()]); - let error = slice_first_axis_range(&tensor, size_of::(), 0, 1, true).unwrap_err(); + let tensor = + WireTensor::from_raw(TensorDtype::F32, vec![2, 2], vec![0; 3 * size_of::()]); + let error = slice_first_axis_range(&tensor, 0, 1, true).unwrap_err(); assert!( - matches!(error, Error::Multimodal(message) if message.contains("expects 16 bytes")) + matches!(error, Error::Multimodal(message) if message.contains("does not match expected 16")) ); } @@ -455,7 +386,7 @@ mod tests { panic!("expected tensor"); }; - assert_eq!(tensor.dtype, "bfloat16"); + assert_eq!(tensor.dtype.as_str(), "bfloat16"); assert_eq!(tensor.shape, vec![2]); assert_eq!(tensor.data.into_raw_view().unwrap().len(), 4); } @@ -470,7 +401,7 @@ mod tests { panic!("expected tensor"); }; - assert_eq!(tensor.dtype, "float16"); + assert_eq!(tensor.dtype.as_str(), "float16"); assert_eq!(tensor.shape, vec![2]); assert_eq!(tensor.data.into_raw_view().unwrap().len(), 4); } diff --git a/rust/src/engine-core-client/src/protocol/dtype.rs b/rust/src/engine-core-client/src/protocol/dtype.rs index 8d321df9d3..2a2b7b6053 100644 --- a/rust/src/engine-core-client/src/protocol/dtype.rs +++ b/rust/src/engine-core-client/src/protocol/dtype.rs @@ -1,7 +1,191 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project +use std::fmt; +use std::str::FromStr; + use serde::{Deserialize, Serialize}; +use serde_with::{DeserializeFromStr, SerializeDisplay}; + +/// Element types accepted by Python's tensor decoder. +#[derive(Debug, Clone, Copy, PartialEq, Eq, SerializeDisplay, DeserializeFromStr)] +pub enum TensorDtype { + Bool, + U8, + I8, + U16, + I16, + U32, + I32, + U64, + I64, + F16, + Bf16, + F32, + F64, +} + +impl TensorDtype { + pub const fn as_str(self) -> &'static str { + match self { + Self::Bool => "bool", + Self::U8 => "uint8", + Self::I8 => "int8", + Self::U16 => "uint16", + Self::I16 => "int16", + Self::U32 => "uint32", + Self::I32 => "int32", + Self::U64 => "uint64", + Self::I64 => "int64", + Self::F16 => "float16", + Self::Bf16 => "bfloat16", + Self::F32 => "float32", + Self::F64 => "float64", + } + } + + pub const fn element_size(self) -> usize { + match self { + Self::Bool | Self::U8 | Self::I8 => 1, + Self::U16 | Self::I16 | Self::F16 | Self::Bf16 => 2, + Self::U32 | Self::I32 | Self::F32 => 4, + Self::U64 | Self::I64 | Self::F64 => 8, + } + } +} + +impl fmt::Display for TensorDtype { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +impl FromStr for TensorDtype { + type Err = String; + + fn from_str(value: &str) -> Result { + match value { + "bool" => Ok(Self::Bool), + "uint8" => Ok(Self::U8), + "int8" => Ok(Self::I8), + "uint16" => Ok(Self::U16), + "int16" => Ok(Self::I16), + "uint32" => Ok(Self::U32), + "int32" => Ok(Self::I32), + "uint64" => Ok(Self::U64), + "int64" => Ok(Self::I64), + "float16" => Ok(Self::F16), + "bfloat16" => Ok(Self::Bf16), + "float32" => Ok(Self::F32), + "float64" => Ok(Self::F64), + _ => Err(format!("unsupported tensor dtype {value:?}")), + } + } +} + +/// Byte order carried by a NumPy dtype descriptor. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Endianness { + Little, + Big, + Native, +} + +/// Numeric array dtype with an explicit byte-order contract. +/// +/// Also accepts Torch type names as native-endian dtypes because prompt +/// logprobs use tensor tuples with the same wire shape as NumPy arrays. +/// Serializes byte order explicitly, including `=` for native byte order. +/// Bfloat16 uses the Torch name after the prefix as a nonstandard descriptor. +#[derive(Debug, Clone, Copy, PartialEq, Eq, SerializeDisplay, DeserializeFromStr)] +pub struct NumpyDtype { + pub scalar: TensorDtype, + pub endianness: Endianness, +} + +impl NumpyDtype { + pub const fn little(scalar: TensorDtype) -> Self { + Self { + scalar, + endianness: Endianness::Little, + } + } + + pub const fn native(scalar: TensorDtype) -> Self { + Self { + scalar, + endianness: Endianness::Native, + } + } + + pub const fn big(scalar: TensorDtype) -> Self { + Self { + scalar, + endianness: Endianness::Big, + } + } +} + +impl From for NumpyDtype { + fn from(scalar: TensorDtype) -> Self { + Self::native(scalar) + } +} + +impl fmt::Display for NumpyDtype { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let prefix = match self.endianness { + Endianness::Native => '=', + Endianness::Little => '<', + Endianness::Big => '>', + }; + let code = match self.scalar { + TensorDtype::Bool => "b1", + TensorDtype::U8 => "u1", + TensorDtype::I8 => "i1", + TensorDtype::U16 => "u2", + TensorDtype::I16 => "i2", + TensorDtype::U32 => "u4", + TensorDtype::I32 => "i4", + TensorDtype::U64 => "u8", + TensorDtype::I64 => "i8", + TensorDtype::F16 => "f2", + TensorDtype::F32 => "f4", + TensorDtype::F64 => "f8", + TensorDtype::Bf16 => "bfloat16", + }; + write!(f, "{prefix}{code}") + } +} + +impl FromStr for NumpyDtype { + type Err = String; + + fn from_str(dtype: &str) -> Result { + let (endianness, body) = match dtype.as_bytes().first().copied() { + Some(b'<') => (Endianness::Little, &dtype[1..]), + Some(b'>') => (Endianness::Big, &dtype[1..]), + Some(b'=' | b'|') => (Endianness::Native, &dtype[1..]), + _ => (Endianness::Native, dtype), + }; + let scalar = match body { + "b1" => TensorDtype::Bool, + "u1" => TensorDtype::U8, + "i1" => TensorDtype::I8, + "u2" => TensorDtype::U16, + "i2" => TensorDtype::I16, + "u4" => TensorDtype::U32, + "i4" => TensorDtype::I32, + "u8" => TensorDtype::U64, + "i8" => TensorDtype::I64, + "f2" => TensorDtype::F16, + "f4" => TensorDtype::F32, + "f8" => TensorDtype::F64, + _ => body.parse().map_err(|_| format!("unsupported dtype string {dtype:?}"))?, + }; + Ok(Self { scalar, endianness }) + } +} /// Effective model dtype reported by the engine after config resolution. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -26,7 +210,53 @@ impl ModelDtype { #[cfg(test)] mod tests { - use super::ModelDtype; + use super::{Endianness, ModelDtype, NumpyDtype, TensorDtype}; + + #[test] + fn numpy_descriptors_preserve_explicit_byte_order() { + for (code, scalar) in [ + ("b1", TensorDtype::Bool), + ("u1", TensorDtype::U8), + ("i1", TensorDtype::I8), + ("u2", TensorDtype::U16), + ("i2", TensorDtype::I16), + ("u4", TensorDtype::U32), + ("i4", TensorDtype::I32), + ("u8", TensorDtype::U64), + ("i8", TensorDtype::I64), + ("f2", TensorDtype::F16), + ("bfloat16", TensorDtype::Bf16), + ("f4", TensorDtype::F32), + ("f8", TensorDtype::F64), + ] { + for (prefix, endianness) in [("<", Endianness::Little), (">", Endianness::Big)] { + let wire = serde_json::json!(format!("{prefix}{code}")); + let dtype: NumpyDtype = serde_json::from_value(wire.clone()).unwrap(); + assert_eq!(dtype, NumpyDtype { scalar, endianness }); + assert_eq!(serde_json::to_value(dtype).unwrap(), wire); + } + for wire in [ + format!("={code}"), + format!("|{code}"), + code.into(), + scalar.as_str().into(), + ] { + let dtype: NumpyDtype = serde_json::from_value(serde_json::json!(wire)).unwrap(); + assert_eq!(dtype, NumpyDtype::from(scalar)); + assert_eq!( + serde_json::to_value(dtype).unwrap(), + serde_json::json!(format!("={code}")) + ); + } + } + } + + #[test] + fn rejects_unsupported_numpy_dtypes() { + for wire in ["", "object", "(serde_json::json!(wire)).is_err()); + } + } #[test] fn serde_uses_protocol_dtype_strings() { diff --git a/rust/src/engine-core-client/src/protocol/logprobs.rs b/rust/src/engine-core-client/src/protocol/logprobs.rs index ee8d4101b7..d5bc40e061 100644 --- a/rust/src/engine-core-client/src/protocol/logprobs.rs +++ b/rust/src/engine-core-client/src/protocol/logprobs.rs @@ -14,6 +14,7 @@ use serde::{Deserialize, Deserializer, Serialize}; use self::wire::*; use crate::error::{Error, Result, bail_ext_value_decode}; +use crate::protocol::dtype::{NumpyDtype, TensorDtype}; use crate::protocol::tensor::{WireArrayData, WireNdArray}; /// One token candidate and its logprob metadata for a single sequence position. @@ -205,17 +206,17 @@ impl WireLogprobs { Ok(Self { logprob_token_ids: WireNdArray { - dtype: " { pub rows: usize, @@ -37,7 +24,7 @@ pub(super) fn decode_array2_u32( frames: &[Bytes], ) -> Result> { let (shape, bytes, scalar, endianness) = - decode_array_metadata(value, field, frames, &[ScalarType::I32, ScalarType::I64])?; + decode_array_metadata(value, field, frames, &[TensorDtype::I32, TensorDtype::I64])?; if shape.len() != 2 { return Err(decode_error( field, @@ -46,15 +33,15 @@ pub(super) fn decode_array2_u32( } let data = match scalar { - ScalarType::I32 => decode_i32_vec(&bytes, endianness, field)? + TensorDtype::I32 => decode_i32_vec(&bytes, endianness, field)? .into_iter() .map(|value| convert_to_u32(value, field)) .try_collect()?, - ScalarType::I64 => decode_i64_vec(&bytes, endianness, field)? + TensorDtype::I64 => decode_i64_vec(&bytes, endianness, field)? .into_iter() .map(|value| convert_to_u32(value, field)) .try_collect()?, - ScalarType::F32 => unreachable!("scalar validation should reject f32"), + _ => unreachable!("scalar validation should accept only i32 and i64"), }; Ok(DecodedArray2 { rows: shape[0], @@ -69,7 +56,7 @@ pub(super) fn decode_array1_u32( frames: &[Bytes], ) -> Result> { let (shape, bytes, scalar, endianness) = - decode_array_metadata(value, field, frames, &[ScalarType::I32, ScalarType::I64])?; + decode_array_metadata(value, field, frames, &[TensorDtype::I32, TensorDtype::I64])?; if shape.len() != 1 { return Err(decode_error( field, @@ -78,15 +65,15 @@ pub(super) fn decode_array1_u32( } let data = match scalar { - ScalarType::I32 => decode_i32_vec(&bytes, endianness, field)? + TensorDtype::I32 => decode_i32_vec(&bytes, endianness, field)? .into_iter() .map(|value| convert_to_u32(value, field)) .try_collect()?, - ScalarType::I64 => decode_i64_vec(&bytes, endianness, field)? + TensorDtype::I64 => decode_i64_vec(&bytes, endianness, field)? .into_iter() .map(|value| convert_to_u32(value, field)) .try_collect()?, - ScalarType::F32 => unreachable!("scalar validation should reject f32"), + _ => unreachable!("scalar validation should accept only i32 and i64"), }; Ok(data) } @@ -97,7 +84,7 @@ pub(super) fn decode_array2_f32( frames: &[Bytes], ) -> Result> { let (shape, bytes, _, endianness) = - decode_array_metadata(value, field, frames, &[ScalarType::F32])?; + decode_array_metadata(value, field, frames, &[TensorDtype::F32])?; if shape.len() != 2 { return Err(decode_error( field, @@ -117,15 +104,16 @@ pub(super) fn decode_array_metadata( value: WireNdArray, field: &str, frames: &[Bytes], - expected_scalars: &[ScalarType], -) -> Result<(Vec, Bytes, ScalarType, Endianness)> { + expected_scalars: &[TensorDtype], +) -> Result<(Vec, Bytes, TensorDtype, Endianness)> { let mut value = value; - let (scalar, endianness) = parse_dtype(&value.dtype, field)?; + let scalar = value.dtype.scalar; + let endianness = value.dtype.endianness; if !expected_scalars.contains(&scalar) { return Err(decode_error( field, &format!( - "expected dtype in {:?}, got {}", + "expected dtype in {:?}, got {:?}", expected_scalars, value.dtype ), )); @@ -140,42 +128,16 @@ pub(super) fn decode_array_metadata( Ok((shape, bytes, scalar, endianness)) } -pub(super) fn parse_dtype(dtype: &str, field: &str) -> Result<(ScalarType, Endianness)> { - let (endianness, body) = match dtype.as_bytes().first().copied() { - Some(b'<') => (Endianness::Little, &dtype[1..]), - Some(b'>') => (Endianness::Big, &dtype[1..]), - Some(b'=') => (Endianness::Native, &dtype[1..]), - Some(b'|') => (Endianness::Native, &dtype[1..]), - _ => (Endianness::Native, dtype), - }; - - let scalar = match body { - "i4" | "int32" => ScalarType::I32, - "i8" | "int64" => ScalarType::I64, - "f4" | "float32" => ScalarType::F32, - _ => { - return Err(decode_error( - field, - &format!("unsupported dtype string {dtype:?}"), - )); - } - }; - Ok((scalar, endianness)) -} - pub(super) fn validate_byte_length( shape: &[usize], byte_len: usize, field: &str, - scalar: ScalarType, + scalar: TensorDtype, ) -> Result<()> { let element_count = shape .checked_numel() .ok_or_else(|| decode_error(field, "shape element count overflowed usize"))?; - let element_size = match scalar { - ScalarType::I32 | ScalarType::F32 => 4, - ScalarType::I64 => 8, - }; + let element_size = scalar.element_size(); let expected = element_count .checked_mul(element_size) .ok_or_else(|| decode_error(field, "byte length overflowed usize"))?; diff --git a/rust/src/engine-core-client/src/protocol/logprobs/tests.rs b/rust/src/engine-core-client/src/protocol/logprobs/tests.rs index 2708637ebc..2470460b52 100644 --- a/rust/src/engine-core-client/src/protocol/logprobs/tests.rs +++ b/rust/src/engine-core-client/src/protocol/logprobs/tests.rs @@ -302,6 +302,29 @@ fn decodes_big_endian_payloads() { ); } +#[test] +fn rejects_supported_array_dtypes_in_incompatible_logprobs_fields() { + for (ids_dtype, probs_dtype, field) in [ + (" &'static str { + match self { + Self::Image => "image", + Self::Audio => "audio", + Self::Video => "video", + } + } +} + /// Multimodal feature payload accepted from higher-level frontend code. /// /// Original Python definition: @@ -32,7 +52,7 @@ pub struct MmFeatureSpec { pub data: Option, /// The input modality, e.g., `"image"`, `"audio"`, `"video"`. - pub modality: String, + pub modality: MmModality, /// The hash for caching encoder outputs (with LoRA prefix if applicable). pub identifier: String, diff --git a/rust/src/engine-core-client/src/protocol/request.rs b/rust/src/engine-core-client/src/protocol/request.rs index 8118b770e3..973ada71c6 100644 --- a/rust/src/engine-core-client/src/protocol/request.rs +++ b/rust/src/engine-core-client/src/protocol/request.rs @@ -158,8 +158,10 @@ mod tests { use rmpv::Value; use super::*; + use crate::protocol::dtype::TensorDtype; use crate::protocol::multimodal::{ - MmBatchedField, MmFeatureSpec, MmField, MmFieldElem, MmKwargValue, PlaceholderRange, + MmBatchedField, MmFeatureSpec, MmField, MmFieldElem, MmKwargValue, MmModality, + PlaceholderRange, }; use crate::protocol::sampling::EngineCoreSamplingParams; use crate::protocol::tensor::{WireArrayData, WireTensor}; @@ -212,7 +214,7 @@ mod tests { "inline".to_string(), MmFieldElem { data: Some(MmKwargValue::Tensor(WireTensor::from_raw( - "uint8", + TensorDtype::U8, vec![inline.len()], inline, ))), @@ -225,7 +227,7 @@ mod tests { data: Some(MmKwargValue::List(vec![ MmKwargValue::Int(7), MmKwargValue::Tensor(WireTensor::from_raw( - "uint8", + TensorDtype::U8, vec![first_aux.len()], first_aux, )), @@ -234,13 +236,13 @@ mod tests { }, ), ])), - modality: "image".to_string(), + modality: MmModality::Image, identifier: "id".to_string(), mm_position: PlaceholderRange { offset: 0, length: second_aux.len(), is_embed: Some(WireTensor::from_raw( - "bool", + TensorDtype::Bool, vec![second_aux.len()], second_aux, )), diff --git a/rust/src/engine-core-client/src/protocol/tensor.rs b/rust/src/engine-core-client/src/protocol/tensor.rs index ac08dd2f2f..2bd193f76b 100644 --- a/rust/src/engine-core-client/src/protocol/tensor.rs +++ b/rust/src/engine-core-client/src/protocol/tensor.rs @@ -6,9 +6,12 @@ use bytes::Bytes; use enum_as_inner::EnumAsInner; use half::{bf16, f16}; use rmpv::Value; +use serde::de::DeserializeOwned; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde_tuple::{Deserialize_tuple, Serialize_tuple}; +use super::dtype::{NumpyDtype, TensorDtype}; + /// Tensors and ndarrays are encoded with this extension type in Python. /// /// Original Python definition: @@ -63,20 +66,20 @@ impl [usize] { /// /// #[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple)] -pub struct WireNdArray { - pub dtype: String, +pub struct WireArray { + pub dtype: Dtype, pub shape: Vec, pub data: WireArrayData, } -impl WireNdArray { +impl + Serialize + DeserializeOwned> WireArray { /// Build a float32 tensor/ndarray backed by native-endian raw-view bytes. /// /// Takes ownership of the backing buffer without copying its data. pub fn from_f32(shape: Vec, data: Vec) -> Result { validate_element_count(&shape, data.len())?; Ok(Self::from_raw_bytes( - "float32", + TensorDtype::F32, shape, bytes_from_pod_vec(data), )) @@ -88,7 +91,7 @@ impl WireNdArray { pub fn from_f16(shape: Vec, data: Vec) -> Result { validate_element_count(&shape, data.len())?; Ok(Self::from_raw_bytes( - "float16", + TensorDtype::F16, shape, bytes_from_pod_vec(data), )) @@ -100,7 +103,7 @@ impl WireNdArray { pub fn from_bf16(shape: Vec, data: Vec) -> Result { validate_element_count(&shape, data.len())?; Ok(Self::from_raw_bytes( - "bfloat16", + TensorDtype::Bf16, shape, bytes_from_pod_vec(data), )) @@ -112,7 +115,7 @@ impl WireNdArray { pub fn from_i64(shape: Vec, data: Vec) -> Result { validate_element_count(&shape, data.len())?; Ok(Self::from_raw_bytes( - "int64", + TensorDtype::I64, shape, bytes_from_pod_vec(data), )) @@ -124,7 +127,7 @@ impl WireNdArray { pub fn from_u32(shape: Vec, data: Vec) -> Result { validate_element_count(&shape, data.len())?; Ok(Self::from_raw_bytes( - "uint32", + TensorDtype::U32, shape, bytes_from_pod_vec(data), )) @@ -137,7 +140,7 @@ impl WireNdArray { pub fn from_bool(shape: Vec, data: Vec) -> Result { validate_element_count(&shape, data.len())?; Ok(Self { - dtype: "bool".to_string(), + dtype: TensorDtype::Bool.into(), shape, data: WireArrayData::RawView(Bytes::from( data.into_iter().map(u8::from).collect::>(), @@ -149,12 +152,12 @@ impl WireNdArray { /// /// Use this as an escape hatch when the caller already owns bytes that /// match the requested `dtype` and `shape`. - pub fn from_raw(dtype: impl Into, shape: Vec, data: Vec) -> Self { + pub fn from_raw(dtype: impl Into, shape: Vec, data: Vec) -> Self { Self::from_raw_bytes(dtype, shape, Bytes::from(data)) } /// Build a tensor/ndarray from an owned immutable raw-view buffer. - pub fn from_raw_bytes(dtype: impl Into, shape: Vec, data: Bytes) -> Self { + pub fn from_raw_bytes(dtype: impl Into, shape: Vec, data: Bytes) -> Self { Self { dtype: dtype.into(), shape, @@ -192,7 +195,10 @@ fn validate_element_count(shape: &[usize], len: usize) -> Result<(), String> { /// /// This is the same wire shape as [`WireNdArray`]; multimodal request payloads /// use it for `torch.Tensor` values. -pub type WireTensor = WireNdArray; +pub type WireTensor = WireArray; + +/// Numeric array with a typed NumPy descriptor or native-endian Torch dtype. +pub type WireNdArray = WireArray; /// Python array/tensor payload reference inside [`WireNdArray`]. /// @@ -211,6 +217,27 @@ pub enum WireArrayData { RawView(Bytes), } +impl WireTensor { + /// Validate the shape and storage of an inline tensor. + pub fn validate_inline(&self) -> Result<(), String> { + let expected = self + .shape + .checked_numel() + .and_then(|numel| numel.checked_mul(self.dtype.element_size())) + .ok_or_else(|| format!("tensor shape or byte length overflows: {:?}", self.shape))?; + match &self.data { + WireArrayData::RawView(bytes) if bytes.len() == expected => Ok(()), + WireArrayData::RawView(bytes) => Err(format!( + "tensor byte length {} does not match expected {expected}", + bytes.len() + )), + WireArrayData::AuxIndex(index) => { + Err(format!("inline tensor references auxiliary frame {index}")) + } + } + } +} + impl WireArrayData { /// Resolve an auxiliary-frame reference into owned raw bytes. pub(crate) fn resolve_aux_frame(&mut self, frames: &[Bytes]) -> Result<(), String> { @@ -288,6 +315,21 @@ impl Serialize for WireArrayData { mod tests { use super::*; + #[test] + fn tensor_and_ndarray_preserve_their_dtype_wire_contracts() { + let tensor = WireTensor::from_f32(vec![1], vec![1.0]).unwrap(); + let bytes = rmp_serde::to_vec_named(&tensor).unwrap(); + let wire: Value = rmp_serde::from_slice(&bytes).unwrap(); + assert_eq!(wire.as_array().unwrap()[0], Value::from("float32")); + + let array: WireNdArray = rmp_serde::from_slice(&bytes).unwrap(); + assert_eq!(array.dtype, NumpyDtype::native(TensorDtype::F32)); + let bytes = rmp_serde::to_vec_named(&array).unwrap(); + let wire: Value = rmp_serde::from_slice(&bytes).unwrap(); + assert_eq!(wire.as_array().unwrap()[0], Value::from("=f4")); + assert!(rmp_serde::from_slice::(&bytes).is_err()); + } + #[test] fn raw_view_serializes_as_msgpack_ext() { let bytes = vec![1, 2, 3, 4]; @@ -307,8 +349,8 @@ mod tests { fn constructors_build_raw_view_tensors() { let f32_data = vec![1.0, 2.5]; let f32_data_ptr = f32_data.as_ptr().cast::(); - let f32_tensor = WireNdArray::from_f32(vec![2], f32_data).unwrap(); - assert_eq!(f32_tensor.dtype, "float32"); + let f32_tensor = WireTensor::from_f32(vec![2], f32_data).unwrap(); + assert_eq!(f32_tensor.dtype, TensorDtype::F32); assert_eq!(f32_tensor.shape, vec![2]); let f32_raw_view = f32_tensor.data.into_raw_view().expect("raw view"); assert_eq!(f32_raw_view.as_ptr(), f32_data_ptr); @@ -318,58 +360,49 @@ mod tests { ); let f16_tensor = - WireNdArray::from_f16(vec![2], vec![f16::from_f32(1.0), f16::from_f32(2.5)]).unwrap(); - assert_eq!(f16_tensor.dtype, "float16"); + WireTensor::from_f16(vec![2], vec![f16::from_f32(1.0), f16::from_f32(2.5)]).unwrap(); + assert_eq!(f16_tensor.dtype, TensorDtype::F16); assert_eq!(f16_tensor.shape, vec![2]); assert_eq!(f16_tensor.data.into_raw_view().expect("raw view").len(), 4); let bf16_tensor = - WireNdArray::from_bf16(vec![2], vec![bf16::from_f32(1.0), bf16::from_f32(2.5)]) - .unwrap(); - assert_eq!(bf16_tensor.dtype, "bfloat16"); + WireTensor::from_bf16(vec![2], vec![bf16::from_f32(1.0), bf16::from_f32(2.5)]).unwrap(); + assert_eq!(bf16_tensor.dtype, TensorDtype::Bf16); assert_eq!(bf16_tensor.shape, vec![2]); assert_eq!(bf16_tensor.data.into_raw_view().expect("raw view").len(), 4); - let i64_tensor = WireNdArray::from_i64(vec![1], vec![-7]).unwrap(); - assert_eq!(i64_tensor.dtype, "int64"); + let i64_tensor = WireTensor::from_i64(vec![1], vec![-7]).unwrap(); + assert_eq!(i64_tensor.dtype, TensorDtype::I64); assert_eq!( i64_tensor.data.into_raw_view().expect("raw view").as_ref(), (-7_i64).to_ne_bytes().as_ref() ); - let u32_tensor = WireNdArray::from_u32(vec![1], vec![42]).unwrap(); - assert_eq!(u32_tensor.dtype, "uint32"); + let u32_tensor = WireTensor::from_u32(vec![1], vec![42]).unwrap(); + assert_eq!(u32_tensor.dtype, TensorDtype::U32); assert_eq!( u32_tensor.data.into_raw_view().expect("raw view").as_ref(), 42_u32.to_ne_bytes().as_ref() ); - let bool_tensor = WireNdArray::from_bool(vec![2], vec![false, true]).unwrap(); - assert_eq!(bool_tensor.dtype, "bool"); + let bool_tensor = WireTensor::from_bool(vec![2], vec![false, true]).unwrap(); + assert_eq!(bool_tensor.dtype, TensorDtype::Bool); assert_eq!( bool_tensor.data.into_raw_view().expect("raw view"), vec![0, 1] ); - - let raw_tensor = WireNdArray::from_raw("custom", vec![3], vec![1, 2, 3]); - assert_eq!(raw_tensor.dtype, "custom"); - assert_eq!(raw_tensor.shape, vec![3]); - assert_eq!( - raw_tensor.data.into_raw_view().expect("raw view"), - vec![1, 2, 3] - ); } #[test] fn constructors_validate_shape_product() { - let err = WireNdArray::from_f32(vec![2, 2], vec![1.0, 2.0]).unwrap_err(); + let err = WireTensor::from_f32(vec![2, 2], vec![1.0, 2.0]).unwrap_err(); assert!(err.contains("does not match shape")); } #[test] fn resolve_aux_frame_shares_bytes_using_one_based_message_index() { - let mut tensor = WireNdArray { - dtype: "float32".to_string(), + let mut tensor = WireTensor { + dtype: TensorDtype::F32, shape: vec![2], data: WireArrayData::AuxIndex(1), }; diff --git a/rust/src/engine-core-client/src/tests/client.rs b/rust/src/engine-core-client/src/tests/client.rs index d2d52c7294..13525375f8 100644 --- a/rust/src/engine-core-client/src/tests/client.rs +++ b/rust/src/engine-core-client/src/tests/client.rs @@ -22,8 +22,8 @@ use zeromq::{DealerSocket, PushSocket, SocketOptions, SubSocket, XPubSocket, Zmq use crate::protocol::handshake::{EngineCoreReadyResponse, HandshakeInitMessage, ReadyMessage}; use crate::protocol::logprobs::MaybeWireLogprobs; use crate::protocol::multimodal::{ - MmFeatureSpec, MmField, MmFieldElem, MmFlatField, MmKwargValue, MmSlice, PlaceholderRange, - SliceSpec, + MmFeatureSpec, MmField, MmFieldElem, MmFlatField, MmKwargValue, MmModality, MmSlice, + PlaceholderRange, SliceSpec, }; use crate::protocol::output::{ DpControlMessage, DpControlOutput, EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, @@ -191,7 +191,7 @@ fn sample_multimodal_request() -> EngineCoreRequest { }), }, )])), - modality: "image".to_string(), + modality: MmModality::Image, identifier: "mm-cache-key".to_string(), mm_position: PlaceholderRange { offset: 1, diff --git a/rust/src/server/src/grpc/tests.rs b/rust/src/server/src/grpc/tests.rs index ed465e5c7c..ea4364480c 100644 --- a/rust/src/server/src/grpc/tests.rs +++ b/rust/src/server/src/grpc/tests.rs @@ -728,7 +728,7 @@ async fn unary_generate_prepares_multimodal_input_for_engine_core() { assert_eq!(features.len(), 2); for (feature, identifier) in features.iter().zip(["image-1", "image-2"]) { - assert_eq!(feature.modality, "image"); + assert_eq!(feature.modality.as_str(), "image"); assert_eq!(feature.identifier, identifier); assert!(feature.mm_position.length > 1); assert_eq!( diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 62ac4c4773..a04c3f2f11 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -2736,7 +2736,7 @@ async fn non_stream_chat_image_url_reaches_engine_mm_features() { let features = request.mm_features.as_ref().expect("multimodal features"); assert_eq!(features.len(), 1); - assert_eq!(features[0].modality, "image"); + assert_eq!(features[0].modality.as_str(), "image"); assert_eq!(features[0].identifier, "image-1"); assert!(features[0].mm_position.length > 0); assert!(features[0].mm_position.is_embed.is_some()); @@ -2799,7 +2799,7 @@ async fn non_stream_chat_rejects_when_image_count_exceeds_limit_mm_per_prompt() default_stream_output_specs(), Arc::new(FakeChatBackend::with_multimodal_model_info( qwen_multimodal_model_info_with_limits(std::collections::HashMap::from([( - vllm_chat::multimodal::MmLimitModality::Image, + vllm_chat::multimodal::MmModality::Image, vllm_chat::multimodal::MmLimitSpec::Count(1), )])), )), diff --git a/rust/src/text/src/lower.rs b/rust/src/text/src/lower.rs index 8e516921e9..0b18fd4238 100644 --- a/rust/src/text/src/lower.rs +++ b/rust/src/text/src/lower.rs @@ -318,7 +318,9 @@ mod tests { use std::collections::{BTreeSet, HashMap}; use serial_test::file_serial; - use vllm_engine_core_client::protocol::multimodal::{MmFeatureSpec, PlaceholderRange}; + use vllm_engine_core_client::protocol::multimodal::{ + MmFeatureSpec, MmModality, PlaceholderRange, + }; use vllm_tokenizer::test_utils::TestTokenizer; use super::*; @@ -720,7 +722,7 @@ mod tests { fn lower_text_request_moves_multimodal_features_to_generate_request() { let features = vec![MmFeatureSpec { data: None, - modality: "image".to_string(), + modality: MmModality::Image, identifier: "image-1".to_string(), mm_position: PlaceholderRange { offset: 2,