From f1f1259692004c909f8b7f8f371d42c7a7871fa3 Mon Sep 17 00:00:00 2001 From: Sage <80211083+sagearc@users.noreply.github.com> Date: Mon, 20 Jul 2026 11:28:25 +0300 Subject: [PATCH] [Rust Frontend] Use zero-copy slicing for multimodal tensors (#48781) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Sage Ahrac --- rust/src/chat/src/multimodal/item.rs | 8 +- rust/src/chat/src/multimodal/tensor.rs | 149 ++++++++---------- rust/src/chat/src/multimodal/video.rs | 8 +- .../engine-core-client/src/protocol/tensor.rs | 25 +-- 4 files changed, 91 insertions(+), 99 deletions(-) diff --git a/rust/src/chat/src/multimodal/item.rs b/rust/src/chat/src/multimodal/item.rs index d5600c5688..69ab71af30 100644 --- a/rust/src/chat/src/multimodal/item.rs +++ b/rust/src/chat/src/multimodal/item.rs @@ -38,7 +38,7 @@ pub(super) fn build_batched_items( let keep_on_cpu = spec.keep_on_cpu_keys.contains(key); let (value, field) = match spec.field_layout_for(key) { Some(FieldLayout::Batched) => ( - tensor.batched_value_at(index)?, + tensor.batched_wire_value_at(index)?, MmField::Batched(MmBatchedField { keep_on_cpu }), ), Some(FieldLayout::Flat { sizes_key }) => { @@ -47,7 +47,7 @@ pub(super) fn build_batched_items( })?; let (start, end) = tensor::flat_range_for_index(sizes, sizes_key, index)?; ( - tensor.flat_value_range(start, end)?, + tensor.flat_wire_value_range(start, end)?, MmField::Flat(MmFlatField { slices: vec![MmSlice::Slice(SliceSpec { start: Some(0), @@ -60,7 +60,7 @@ pub(super) fn build_batched_items( ) } None => ( - tensor.clone(), + tensor.try_into()?, MmField::Shared(MmSharedField { batch_size: len, keep_on_cpu, @@ -71,7 +71,7 @@ pub(super) fn build_batched_items( data.insert( key.clone(), MmFieldElem { - data: Some(value.try_into()?), + data: Some(value), field, }, ); diff --git a/rust/src/chat/src/multimodal/tensor.rs b/rust/src/chat/src/multimodal/tensor.rs index f701e0646c..35b57dd5fe 100644 --- a/rust/src/chat/src/multimodal/tensor.rs +++ b/rust/src/chat/src/multimodal/tensor.rs @@ -12,7 +12,7 @@ use vllm_engine_core_client::protocol::tensor::{ShapeExt as _, WireTensor}; use crate::error::{Error, Result, bail_multimodal, multimodal}; /// Representation for multimodal kwarg values for transformation. -#[derive(Debug, Clone)] +#[derive(Debug)] pub(super) enum KwargValue { /// Float tensor with row-major flat data and shape. F32Tensor { data: Vec, shape: Vec }, @@ -107,28 +107,19 @@ impl KwargValue { } } -impl TryFrom for ProtocolKwargValue { +impl TryFrom<&KwargValue> for ProtocolKwargValue { type Error = Error; - fn try_from(value: KwargValue) -> Result { - match value { - KwargValue::F32Tensor { data, shape } => Ok(Self::Tensor( - WireTensor::from_f32(shape, data).map_err(Error::Multimodal)?, - )), - KwargValue::F16Tensor { data, shape } => Ok(Self::Tensor( - WireTensor::from_f16(shape, data).map_err(Error::Multimodal)?, - )), - KwargValue::Bf16Tensor { data, shape } => Ok(Self::Tensor( - WireTensor::from_bf16(shape, data).map_err(Error::Multimodal)?, - )), - KwargValue::I64Tensor { data, shape } => Ok(Self::Tensor( - WireTensor::from_i64(shape, data).map_err(Error::Multimodal)?, - )), - KwargValue::U32Tensor { data, shape } => Ok(Self::Tensor( - WireTensor::from_u32(shape, data).map_err(Error::Multimodal)?, - )), - KwargValue::Passthrough(value) => Ok(value), - } + fn try_from(value: &KwargValue) -> Result { + let tensor = match value { + KwargValue::F32Tensor { data, shape } => WireTensor::from_f32(shape.clone(), data), + KwargValue::F16Tensor { data, shape } => WireTensor::from_f16(shape.clone(), data), + KwargValue::Bf16Tensor { data, shape } => WireTensor::from_bf16(shape.clone(), data), + KwargValue::I64Tensor { data, shape } => WireTensor::from_i64(shape.clone(), data), + KwargValue::U32Tensor { data, shape } => WireTensor::from_u32(shape.clone(), data), + KwargValue::Passthrough(value) => return Ok(value.clone()), + }; + tensor.map(ProtocolKwargValue::Tensor).map_err(Error::Multimodal) } } @@ -145,63 +136,55 @@ impl KwargValue { } } - /// Extract one media item from a batched tensor field. + /// Convert one media item from a batched tensor field to wire bytes. /// /// Batched fields use their first axis as media-item index and drop that /// axis in the per-feature value, matching vLLM's batched-field semantics. - pub(super) fn batched_value_at(&self, index: usize) -> Result { - match self { - Self::F32Tensor { data, shape } => { - let (shape, data) = slice_first_axis_range(shape, data, index, index + 1, true)?; - Ok(Self::F32Tensor { data, shape }) - } - Self::F16Tensor { data, shape } => { - let (shape, data) = slice_first_axis_range(shape, data, index, index + 1, true)?; - Ok(Self::F16Tensor { data, shape }) - } - Self::Bf16Tensor { data, shape } => { - let (shape, data) = slice_first_axis_range(shape, data, index, index + 1, true)?; - Ok(Self::Bf16Tensor { data, shape }) - } - Self::I64Tensor { data, shape } => { - let (shape, data) = slice_first_axis_range(shape, data, index, index + 1, true)?; - Ok(Self::I64Tensor { data, shape }) - } - Self::U32Tensor { data, shape } => { - let (shape, data) = slice_first_axis_range(shape, data, index, index + 1, true)?; - Ok(Self::U32Tensor { data, shape }) - } - Self::Passthrough(value) => Ok(Self::Passthrough(value.clone())), - } + pub(super) fn batched_wire_value_at(&self, index: usize) -> Result { + self.wire_value_range(index, index + 1, true) } - /// Extract one media item's variable-length range from a flat tensor field. + /// Convert one media item's flat tensor range directly to wire bytes. /// /// Flat fields keep the first axis as the sliced length for this item. - pub(super) fn flat_value_range(&self, start: usize, end: usize) -> Result { - match self { + pub(super) fn flat_wire_value_range( + &self, + start: usize, + end: usize, + ) -> Result { + self.wire_value_range(start, end, false) + } + + fn wire_value_range( + &self, + start: usize, + end: usize, + drop_axis: bool, + ) -> Result { + let tensor = match self { Self::F32Tensor { data, shape } => { - let (shape, data) = slice_first_axis_range(shape, data, start, end, false)?; - Ok(Self::F32Tensor { data, shape }) + let (shape, data) = slice_first_axis_range(shape, data, start, end, drop_axis)?; + WireTensor::from_f32(shape, data) } Self::F16Tensor { data, shape } => { - let (shape, data) = slice_first_axis_range(shape, data, start, end, false)?; - Ok(Self::F16Tensor { data, shape }) + let (shape, data) = slice_first_axis_range(shape, data, start, end, drop_axis)?; + WireTensor::from_f16(shape, data) } Self::Bf16Tensor { data, shape } => { - let (shape, data) = slice_first_axis_range(shape, data, start, end, false)?; - Ok(Self::Bf16Tensor { data, shape }) + let (shape, data) = slice_first_axis_range(shape, data, start, end, drop_axis)?; + WireTensor::from_bf16(shape, data) } Self::I64Tensor { data, shape } => { - let (shape, data) = slice_first_axis_range(shape, data, start, end, false)?; - Ok(Self::I64Tensor { data, shape }) + let (shape, data) = slice_first_axis_range(shape, data, start, end, drop_axis)?; + WireTensor::from_i64(shape, data) } Self::U32Tensor { data, shape } => { - let (shape, data) = slice_first_axis_range(shape, data, start, end, false)?; - Ok(Self::U32Tensor { data, shape }) + let (shape, data) = slice_first_axis_range(shape, data, start, end, drop_axis)?; + WireTensor::from_u32(shape, data) } - Self::Passthrough(value) => Ok(Self::Passthrough(value.clone())), - } + Self::Passthrough(value) => return Ok(value.clone()), + }; + tensor.map(ProtocolKwargValue::Tensor).map_err(Error::Multimodal) } } @@ -240,13 +223,13 @@ fn tensor_as_usize_vec(tensor: &KwargValue) -> Result> { } /// Slice a flat row-major tensor along its first axis. -fn slice_first_axis_range( +fn slice_first_axis_range<'a, T>( shape: &[usize], - data: &[T], + data: &'a [T], start: usize, end: usize, drop_axis: bool, -) -> Result<(Vec, Vec)> { +) -> Result<(Vec, &'a [T])> { let first_dim = *shape.first().ok_or_else(|| multimodal!("tensor has no first dimension"))?; if start > end || end > first_dim { bail_multimodal!("invalid tensor slice {start}..{end} for first dimension {first_dim}"); @@ -270,7 +253,7 @@ fn slice_first_axis_range( shape[0] = end - start; shape }; - Ok((out_shape, data[data_start..data_end].to_vec())) + Ok((out_shape, &data[data_start..data_end])) } #[cfg(test)] @@ -278,35 +261,39 @@ mod tests { use super::*; #[test] - fn batched_value_at_drops_first_axis() { + fn batched_wire_value_at_drops_first_axis() { let value = KwargValue::F32Tensor { data: vec![1.0, 2.0, 3.0, 4.0], shape: vec![2, 2], }; - let value = value.batched_value_at(1).unwrap(); + let ProtocolKwargValue::Tensor(tensor) = value.batched_wire_value_at(1).unwrap() else { + panic!("expected tensor"); + }; - assert!(matches!( - value, - KwargValue::F32Tensor { data, shape } - if shape == vec![2] && data == vec![3.0, 4.0] - )); + assert_eq!(tensor.shape, vec![2]); + assert_eq!( + tensor.data.into_raw_view().unwrap(), + [3.0_f32, 4.0].into_iter().flat_map(f32::to_ne_bytes).collect::>() + ); } #[test] - fn flat_value_range_keeps_first_axis() { + fn flat_wire_value_range_keeps_first_axis() { let value = KwargValue::U32Tensor { data: (0..10).collect(), shape: vec![5, 2], }; - let value = value.flat_value_range(1, 3).unwrap(); + let ProtocolKwargValue::Tensor(tensor) = value.flat_wire_value_range(1, 3).unwrap() else { + panic!("expected tensor"); + }; - assert!(matches!( - value, - KwargValue::U32Tensor { data, shape } - if shape == vec![2, 2] && data == vec![2, 3, 4, 5] - )); + assert_eq!(tensor.shape, vec![2, 2]); + assert_eq!( + tensor.data.into_raw_view().unwrap(), + [2_u32, 3, 4, 5].into_iter().flat_map(u32::to_ne_bytes).collect::>() + ); } #[test] @@ -336,7 +323,7 @@ mod tests { let value = KwargValue::from_f32_tensor(vec![1.0, -1.0], vec![2], ModelDtype::BFloat16).unwrap(); - let ProtocolKwargValue::Tensor(tensor) = ProtocolKwargValue::try_from(value).unwrap() + let ProtocolKwargValue::Tensor(tensor) = ProtocolKwargValue::try_from(&value).unwrap() else { panic!("expected tensor"); }; @@ -351,7 +338,7 @@ mod tests { let value = KwargValue::from_f32_tensor(vec![1.0, -1.0], vec![2], ModelDtype::Float16).unwrap(); - let ProtocolKwargValue::Tensor(tensor) = ProtocolKwargValue::try_from(value).unwrap() + let ProtocolKwargValue::Tensor(tensor) = ProtocolKwargValue::try_from(&value).unwrap() else { panic!("expected tensor"); }; diff --git a/rust/src/chat/src/multimodal/video.rs b/rust/src/chat/src/multimodal/video.rs index 9cc3238ac6..6c9b961e7c 100644 --- a/rust/src/chat/src/multimodal/video.rs +++ b/rust/src/chat/src/multimodal/video.rs @@ -130,7 +130,7 @@ fn build_video_item( let keep_on_cpu = support.spec.keep_on_cpu_keys.contains(&key); let (value, field) = match support.spec.field_layout_for(&key) { Some(FieldLayout::Batched) => ( - tensor.batched_value_at(0)?, + tensor.batched_wire_value_at(0)?, MmField::Batched(MmBatchedField { keep_on_cpu }), ), Some(FieldLayout::Flat { .. }) => { @@ -138,7 +138,7 @@ fn build_video_item( .first_dim() .ok_or_else(|| multimodal!("flat video input `{key}` is not a tensor"))?; ( - tensor, + (&tensor).try_into()?, MmField::Flat(MmFlatField { slices: vec![MmSlice::Slice(SliceSpec { start: Some(0), @@ -151,7 +151,7 @@ fn build_video_item( ) } None => ( - tensor, + (&tensor).try_into()?, MmField::Shared(MmSharedField { batch_size: 1, keep_on_cpu, @@ -162,7 +162,7 @@ fn build_video_item( data.insert( key, MmFieldElem { - data: Some(value.try_into()?), + data: Some(value), field, }, ); diff --git a/rust/src/engine-core-client/src/protocol/tensor.rs b/rust/src/engine-core-client/src/protocol/tensor.rs index 5eb1668e75..519bc238de 100644 --- a/rust/src/engine-core-client/src/protocol/tensor.rs +++ b/rust/src/engine-core-client/src/protocol/tensor.rs @@ -55,52 +55,57 @@ pub struct WireNdArray { impl WireNdArray { /// Build a float32 tensor/ndarray backed by native-endian raw-view bytes. - pub fn from_f32(shape: Vec, data: Vec) -> Result { + pub fn from_f32(shape: Vec, data: impl AsRef<[f32]>) -> Result { + let data = data.as_ref(); validate_element_count(&shape, data.len())?; Ok(Self { dtype: "float32".to_string(), shape, - data: WireArrayData::RawView(pod_collect_to_vec::(&data)), + data: WireArrayData::RawView(pod_collect_to_vec::(data)), }) } /// Build a float16 tensor/ndarray backed by native-endian raw-view bytes. - pub fn from_f16(shape: Vec, data: Vec) -> Result { + pub fn from_f16(shape: Vec, data: impl AsRef<[f16]>) -> Result { + let data = data.as_ref(); validate_element_count(&shape, data.len())?; Ok(Self { dtype: "float16".to_string(), shape, - data: WireArrayData::RawView(pod_collect_to_vec::(&data)), + data: WireArrayData::RawView(pod_collect_to_vec::(data)), }) } /// Build a bfloat16 tensor/ndarray backed by native-endian raw-view bytes. - pub fn from_bf16(shape: Vec, data: Vec) -> Result { + pub fn from_bf16(shape: Vec, data: impl AsRef<[bf16]>) -> Result { + let data = data.as_ref(); validate_element_count(&shape, data.len())?; Ok(Self { dtype: "bfloat16".to_string(), shape, - data: WireArrayData::RawView(pod_collect_to_vec::(&data)), + data: WireArrayData::RawView(pod_collect_to_vec::(data)), }) } /// Build an int64 tensor/ndarray backed by native-endian raw-view bytes. - pub fn from_i64(shape: Vec, data: Vec) -> Result { + pub fn from_i64(shape: Vec, data: impl AsRef<[i64]>) -> Result { + let data = data.as_ref(); validate_element_count(&shape, data.len())?; Ok(Self { dtype: "int64".to_string(), shape, - data: WireArrayData::RawView(pod_collect_to_vec::(&data)), + data: WireArrayData::RawView(pod_collect_to_vec::(data)), }) } /// Build a uint32 tensor/ndarray backed by native-endian raw-view bytes. - pub fn from_u32(shape: Vec, data: Vec) -> Result { + pub fn from_u32(shape: Vec, data: impl AsRef<[u32]>) -> Result { + let data = data.as_ref(); validate_element_count(&shape, data.len())?; Ok(Self { dtype: "uint32".to_string(), shape, - data: WireArrayData::RawView(pod_collect_to_vec::(&data)), + data: WireArrayData::RawView(pod_collect_to_vec::(data)), }) }