From 242c591d5a38be6c80ff5197e83e4fa34c5f74b5 Mon Sep 17 00:00:00 2001 From: Reid <61492567+reidliu41@users.noreply.github.com> Date: Wed, 29 Jul 2026 20:21:09 +0800 Subject: [PATCH] [Rust Frontend] Send multimodal tensors in auxiliary frames (#49341) Co-authored-by: Bugen Zhao Signed-off-by: reidliu41 Signed-off-by: Bugen Zhao --- rust/src/chat/src/multimodal/expand.rs | 4 +- rust/src/chat/src/multimodal/tensor.rs | 252 +++++++++++------- rust/src/engine-core-client/src/client.rs | 2 +- rust/src/engine-core-client/src/client/imp.rs | 44 ++- .../src/protocol/logprobs.rs | 6 +- .../src/protocol/logprobs/array.rs | 7 +- .../src/protocol/multimodal.rs | 39 +++ .../src/protocol/request.rs | 97 +++++++ .../engine-core-client/src/protocol/tensor.rs | 142 +++++++--- .../engine-core-client/src/tests/client.rs | 82 +++++- rust/src/engine-core-client/src/transport.rs | 14 +- 11 files changed, 530 insertions(+), 159 deletions(-) diff --git a/rust/src/chat/src/multimodal/expand.rs b/rust/src/chat/src/multimodal/expand.rs index fc4b47f4d2..e449de5909 100644 --- a/rust/src/chat/src/multimodal/expand.rs +++ b/rust/src/chat/src/multimodal/expand.rs @@ -222,7 +222,9 @@ mod tests { assert_eq!(tensor.shape, vec![expected.len()]); assert_eq!( tensor.data, - WireArrayData::RawView(expected.iter().map(|value| u8::from(*value)).collect()) + WireArrayData::RawView( + expected.iter().map(|value| u8::from(*value)).collect::>().into(), + ) ); } diff --git a/rust/src/chat/src/multimodal/tensor.rs b/rust/src/chat/src/multimodal/tensor.rs index 35b57dd5fe..41f0347a41 100644 --- a/rust/src/chat/src/multimodal/tensor.rs +++ b/rust/src/chat/src/multimodal/tensor.rs @@ -2,28 +2,58 @@ // SPDX-FileCopyrightText: Copyright contributors to the vLLM project use std::collections::HashMap; +use std::mem::size_of; use half::{bf16, f16}; use llm_multimodal::{ModelSpecificValue, PreprocessedEncoderInputs}; use vllm_engine_core_client::protocol::dtype::ModelDtype; use vllm_engine_core_client::protocol::multimodal::MmKwargValue as ProtocolKwargValue; -use vllm_engine_core_client::protocol::tensor::{ShapeExt as _, WireTensor}; +use vllm_engine_core_client::protocol::tensor::{ShapeExt as _, WireArrayData, WireTensor}; use crate::error::{Error, Result, bail_multimodal, multimodal}; +/// Element type retained alongside an encoded tensor during multimodal lowering. +#[derive(Debug, Clone, Copy)] +pub(super) enum TensorKind { + /// 32-bit floating point. + F32, + /// IEEE 16-bit floating point. + F16, + /// Brain floating point. + Bf16, + /// Signed 64-bit integer. + I64, + /// Unsigned 32-bit integer. + U32, +} + +impl TensorKind { + const fn element_size(self) -> usize { + match self { + Self::F32 => size_of::(), + Self::F16 => size_of::(), + Self::Bf16 => size_of::(), + Self::I64 => size_of::(), + Self::U32 => size_of::(), + } + } + + const fn wire_dtype(self) -> &'static str { + match self { + Self::F32 => "float32", + Self::F16 => "float16", + Self::Bf16 => "bfloat16", + Self::I64 => "int64", + Self::U32 => "uint32", + } + } +} + /// Representation for multimodal kwarg values for transformation. #[derive(Debug)] pub(super) enum KwargValue { - /// Float tensor with row-major flat data and shape. - F32Tensor { data: Vec, shape: Vec }, - /// Float16 tensor with row-major flat data and shape. - F16Tensor { data: Vec, shape: Vec }, - /// BFloat16 tensor with row-major flat data and shape. - Bf16Tensor { data: Vec, shape: Vec }, - /// Signed integer tensor with row-major flat data and shape. - I64Tensor { data: Vec, shape: Vec }, - /// Unsigned integer tensor with row-major flat data and shape. - U32Tensor { data: Vec, shape: Vec }, + /// Tensor with row-major flat data and shape. + Tensor { kind: TensorKind, wire: WireTensor }, /// Non-tensor kwarg value that is shared or copied as-is. Passthrough(ProtocolKwargValue), } @@ -67,8 +97,14 @@ impl KwargValue { ModelSpecificValue::Tensor { data, shape } => { Self::from_f32_tensor(data, shape, float_dtype)? } - ModelSpecificValue::IntTensor { data, shape } => Self::I64Tensor { data, shape }, - ModelSpecificValue::UintTensor { data, shape } => Self::U32Tensor { data, shape }, + ModelSpecificValue::IntTensor { data, shape } => { + let wire = WireTensor::from_i64(shape, data).map_err(Error::Multimodal)?; + Self::tensor(TensorKind::I64, wire) + } + ModelSpecificValue::UintTensor { data, shape } => { + let wire = WireTensor::from_u32(shape, data).map_err(Error::Multimodal)?; + Self::tensor(TensorKind::U32, wire) + } ModelSpecificValue::Int(value) => Self::Passthrough(Int(value)), ModelSpecificValue::Float(value) => Self::Passthrough(Float(value)), ModelSpecificValue::IntVec(values) => { @@ -93,17 +129,23 @@ impl KwargValue { /// Convert a float tensor to the target float dtype if needed, keeping the /// same shape. fn from_f32_tensor(data: Vec, shape: Vec, float_dtype: ModelDtype) -> Result { - match float_dtype { - ModelDtype::Float16 => Ok(Self::F16Tensor { - data: data.into_iter().map(f16::from_f32).collect(), - shape, - }), - ModelDtype::BFloat16 => Ok(Self::Bf16Tensor { - data: data.into_iter().map(bf16::from_f32).collect(), - shape, - }), - ModelDtype::Float32 => Ok(Self::F32Tensor { data, shape }), - } + let (kind, wire) = match float_dtype { + ModelDtype::Float16 => ( + TensorKind::F16, + WireTensor::from_f16(shape, data.into_iter().map(f16::from_f32).collect()), + ), + ModelDtype::BFloat16 => ( + TensorKind::Bf16, + WireTensor::from_bf16(shape, data.into_iter().map(bf16::from_f32).collect()), + ), + ModelDtype::Float32 => (TensorKind::F32, WireTensor::from_f32(shape, data)), + }; + wire.map(|wire| Self::tensor(kind, wire)).map_err(Error::Multimodal) + } + + fn tensor(kind: TensorKind, wire: WireTensor) -> Self { + debug_assert_eq!(wire.dtype, kind.wire_dtype()); + Self::Tensor { kind, wire } } } @@ -111,15 +153,11 @@ impl TryFrom<&KwargValue> for ProtocolKwargValue { type Error = Error; fn try_from(value: &KwargValue) -> Result { - let tensor = match value { - KwargValue::F32Tensor { data, shape } => WireTensor::from_f32(shape.clone(), data), - KwargValue::F16Tensor { data, shape } => WireTensor::from_f16(shape.clone(), data), - KwargValue::Bf16Tensor { data, shape } => WireTensor::from_bf16(shape.clone(), data), - KwargValue::I64Tensor { data, shape } => WireTensor::from_i64(shape.clone(), data), - KwargValue::U32Tensor { data, shape } => WireTensor::from_u32(shape.clone(), data), + let wire = match value { + KwargValue::Tensor { wire, .. } => wire.clone(), KwargValue::Passthrough(value) => return Ok(value.clone()), }; - tensor.map(ProtocolKwargValue::Tensor).map_err(Error::Multimodal) + Ok(ProtocolKwargValue::Tensor(wire)) } } @@ -127,11 +165,7 @@ impl KwargValue { /// First-axis length for tensor values; `None` for passthrough kwargs. pub(super) fn first_dim(&self) -> Option { match self { - Self::F32Tensor { shape, .. } - | Self::F16Tensor { shape, .. } - | Self::Bf16Tensor { shape, .. } - | Self::I64Tensor { shape, .. } - | Self::U32Tensor { shape, .. } => shape.first().copied(), + Self::Tensor { wire, .. } => wire.shape.first().copied(), Self::Passthrough(_) => None, } } @@ -161,30 +195,13 @@ impl KwargValue { end: usize, drop_axis: bool, ) -> Result { - let tensor = match self { - Self::F32Tensor { data, shape } => { - let (shape, data) = slice_first_axis_range(shape, data, start, end, drop_axis)?; - WireTensor::from_f32(shape, data) - } - Self::F16Tensor { data, shape } => { - let (shape, data) = slice_first_axis_range(shape, data, start, end, drop_axis)?; - WireTensor::from_f16(shape, data) - } - Self::Bf16Tensor { data, shape } => { - let (shape, data) = slice_first_axis_range(shape, data, start, end, drop_axis)?; - WireTensor::from_bf16(shape, data) - } - Self::I64Tensor { data, shape } => { - let (shape, data) = slice_first_axis_range(shape, data, start, end, drop_axis)?; - WireTensor::from_i64(shape, data) - } - Self::U32Tensor { data, shape } => { - let (shape, data) = slice_first_axis_range(shape, data, start, end, drop_axis)?; - WireTensor::from_u32(shape, data) + let wire = match self { + Self::Tensor { kind, wire } => { + slice_first_axis_range(wire, kind.element_size(), start, end, drop_axis) } Self::Passthrough(value) => return Ok(value.clone()), }; - tensor.map(ProtocolKwargValue::Tensor).map_err(Error::Multimodal) + wire.map(ProtocolKwargValue::Tensor) } } @@ -208,44 +225,53 @@ pub(super) fn flat_range_for_index( /// Read a tensor value as per-image sizes for flat slicing. fn tensor_as_usize_vec(tensor: &KwargValue) -> Result> { match tensor { - KwargValue::I64Tensor { data, .. } => data - .iter() + KwargValue::Tensor { + kind: TensorKind::I64, + wire, + } => raw_tensor_bytes(wire, size_of::())? + .chunks_exact(size_of::()) + .map(|bytes| i64::from_ne_bytes(bytes.try_into().expect("exact int64 chunk"))) .map(|value| { - usize::try_from(*value) + usize::try_from(value) .map_err(|_| multimodal!("negative flat tensor size `{value}`")) }) .collect(), - KwargValue::U32Tensor { data, .. } => { - Ok(data.iter().map(|value| *value as usize).collect()) - } + KwargValue::Tensor { + kind: TensorKind::U32, + wire, + } => Ok(raw_tensor_bytes(wire, size_of::())? + .chunks_exact(size_of::()) + .map(|bytes| u32::from_ne_bytes(bytes.try_into().expect("exact uint32 chunk")) as usize) + .collect()), _ => Err(multimodal!("flat tensor sizes must be int64 or uint32")), } } /// Slice a flat row-major tensor along its first axis. -fn slice_first_axis_range<'a, T>( - shape: &[usize], - data: &'a [T], +fn slice_first_axis_range( + tensor: &WireTensor, + element_size: usize, start: usize, end: usize, drop_axis: bool, -) -> Result<(Vec, &'a [T])> { +) -> Result { + let shape = tensor.shape.as_slice(); + raw_tensor_bytes(tensor, element_size)?; let first_dim = *shape.first().ok_or_else(|| multimodal!("tensor has no first dimension"))?; if start > end || end > first_dim { bail_multimodal!("invalid tensor slice {start}..{end} for first dimension {first_dim}"); } - let expected_len = shape - .checked_numel() - .ok_or_else(|| multimodal!("tensor shape {shape:?} has too many elements"))?; - if expected_len != data.len() { - bail_multimodal!( - "tensor shape {shape:?} expects {expected_len} elements, got {}", - data.len() - ); - } - let stride = shape[1..].iter().product::(); - let data_start = start * stride; - let data_end = end * stride; + let stride = shape[1..] + .iter() + .try_fold(1usize, |acc, dim| acc.checked_mul(*dim)) + .and_then(|stride| stride.checked_mul(element_size)) + .ok_or_else(|| multimodal!("tensor shape {shape:?} byte stride overflowed usize"))?; + let data_start = start + .checked_mul(stride) + .ok_or_else(|| multimodal!("tensor slice start byte offset overflowed usize"))?; + let data_end = end + .checked_mul(stride) + .ok_or_else(|| multimodal!("tensor slice end byte offset overflowed usize"))?; let out_shape = if drop_axis { shape[1..].to_vec() } else { @@ -253,7 +279,38 @@ fn slice_first_axis_range<'a, T>( shape[0] = end - start; shape }; - Ok((out_shape, &data[data_start..data_end])) + let WireArrayData::RawView(data) = &tensor.data else { + return Err(multimodal!("cannot slice an aux tensor buffer")); + }; + Ok(WireTensor::from_raw_bytes( + tensor.dtype.clone(), + out_shape, + data.slice(data_start..data_end), + )) +} + +fn raw_tensor_bytes(tensor: &WireTensor, element_size: usize) -> Result<&[u8]> { + let WireArrayData::RawView(data) = &tensor.data else { + return Err(multimodal!("expected an inline tensor buffer")); + }; + let expected_bytes = tensor + .shape + .checked_numel() + .and_then(|numel| numel.checked_mul(element_size)) + .ok_or_else(|| { + multimodal!( + "tensor shape {:?} byte length overflowed usize", + tensor.shape + ) + })?; + if expected_bytes != data.len() { + bail_multimodal!( + "tensor shape {:?} expects {expected_bytes} bytes, got {}", + tensor.shape, + data.len() + ); + } + Ok(data) } #[cfg(test)] @@ -262,28 +319,32 @@ mod tests { #[test] fn batched_wire_value_at_drops_first_axis() { - let value = KwargValue::F32Tensor { - data: vec![1.0, 2.0, 3.0, 4.0], - shape: vec![2, 2], - }; + let data = vec![1.0_f32, 2.0, 3.0, 4.0]; + let expected_ptr = data.as_ptr().cast::().wrapping_add(2 * size_of::()); + let value = KwargValue::tensor( + TensorKind::F32, + WireTensor::from_f32(vec![2, 2], data).unwrap(), + ); let ProtocolKwargValue::Tensor(tensor) = value.batched_wire_value_at(1).unwrap() else { panic!("expected tensor"); }; assert_eq!(tensor.shape, vec![2]); + let raw_view = tensor.data.into_raw_view().unwrap(); + assert_eq!(raw_view.as_ptr(), expected_ptr); assert_eq!( - tensor.data.into_raw_view().unwrap(), + raw_view, [3.0_f32, 4.0].into_iter().flat_map(f32::to_ne_bytes).collect::>() ); } #[test] fn flat_wire_value_range_keeps_first_axis() { - let value = KwargValue::U32Tensor { - data: (0..10).collect(), - shape: vec![5, 2], - }; + let value = KwargValue::tensor( + TensorKind::U32, + WireTensor::from_u32(vec![5, 2], (0..10_u32).collect()).unwrap(), + ); let ProtocolKwargValue::Tensor(tensor) = value.flat_wire_value_range(1, 3).unwrap() else { panic!("expected tensor"); @@ -298,10 +359,10 @@ mod tests { #[test] fn flat_range_for_index_uses_size_tensor() { - let sizes = KwargValue::I64Tensor { - data: vec![2, 3, 4], - shape: vec![3], - }; + let sizes = KwargValue::tensor( + TensorKind::I64, + WireTensor::from_i64(vec![3], vec![2_i64, 3, 4]).unwrap(), + ); assert_eq!( flat_range_for_index(&sizes, "image_grid_thw", 1).unwrap(), @@ -311,10 +372,11 @@ mod tests { #[test] fn slice_first_axis_range_errors_on_shape_data_mismatch() { - let error = slice_first_axis_range(&[2, 2], &[1.0_f32, 2.0, 3.0], 0, 1, true).unwrap_err(); + let tensor = WireTensor::from_raw("float32", vec![2, 2], vec![0; 3 * size_of::()]); + let error = slice_first_axis_range(&tensor, size_of::(), 0, 1, true).unwrap_err(); assert!( - matches!(error, Error::Multimodal(message) if message.contains("expects 4 elements")) + matches!(error, Error::Multimodal(message) if message.contains("expects 16 bytes")) ); } diff --git a/rust/src/engine-core-client/src/client.rs b/rust/src/engine-core-client/src/client.rs index 705cb003e9..a5709e52d6 100644 --- a/rust/src/engine-core-client/src/client.rs +++ b/rust/src/engine-core-client/src/client.rs @@ -519,7 +519,7 @@ impl EngineCoreClient { "registered request to engine" ); - self.inner.send_to_engine(&engine_id, EngineCoreRequestType::Add, &req).await?; + self.inner.send_request_to_engine(&engine_id, req).await?; Ok(()) } .await; diff --git a/rust/src/engine-core-client/src/client/imp.rs b/rust/src/engine-core-client/src/client/imp.rs index e2a1b1b3d8..9c539bf180 100644 --- a/rust/src/engine-core-client/src/client/imp.rs +++ b/rust/src/engine-core-client/src/client/imp.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; use arc_swap::ArcSwapOption; +use bytes::Bytes; use parking_lot::Mutex; use thiserror_ext::AsReport as _; use tokio::runtime::Handle; @@ -21,17 +22,30 @@ use crate::error::{client_closed, dispatcher_closed, unexpected_dispatcher_outpu use crate::metrics::{LoraInfoExporter, SchedulerStatsRecorder}; use crate::protocol::encode_msgpack; use crate::protocol::output::{EngineCoreOutput, EngineCoreOutputs}; -use crate::protocol::request::EngineCoreRequestType; +use crate::protocol::request::{EngineCoreRequest, EngineCoreRequestType}; use crate::protocol::stats::SchedulerStats; use crate::protocol::utility::UtilityOutput; use crate::transport::{ConnectedEngine, EngineId}; use crate::{Error, Result, transport}; +const MSGPACK_ZERO_COPY_THRESHOLD_ENV: &str = "VLLM_MSGPACK_ZERO_COPY_THRESHOLD"; +const DEFAULT_MSGPACK_ZERO_COPY_THRESHOLD: usize = 256; + +fn msgpack_zero_copy_threshold() -> usize { + std::env::var(MSGPACK_ZERO_COPY_THRESHOLD_ENV) + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(DEFAULT_MSGPACK_ZERO_COPY_THRESHOLD) +} + pub(crate) struct ClientInner { input_send: RouterSendHalf, /// The runtime handle used for sending messages to the engine. handle: Handle, model_name: String, + /// Per-tensor byte threshold loaded from env variable + /// `VLLM_MSGPACK_ZERO_COPY_THRESHOLD` when this inner client is created. + msgpack_zero_copy_threshold: usize, scheduler_stats_recorder: SchedulerStatsRecorder, request_reg: Mutex, utility_reg: Mutex, @@ -54,6 +68,7 @@ impl ClientInner { input_send, handle, model_name, + msgpack_zero_copy_threshold: msgpack_zero_copy_threshold(), scheduler_stats_recorder, request_reg: Mutex::new(RequestRegistry::new(engines)), utility_reg: Mutex::new(UtilityRegistry::default()), @@ -229,9 +244,29 @@ impl ClientInner { where T: serde::Serialize + std::fmt::Debug, { - // TODO: for `EngineCoreRequest`, split outbound tensor raw views into aux - // frames instead of always producing a single msgpack frame. - let payload = encode_msgpack(payload)?; + let payload = Bytes::from(encode_msgpack(payload)?); + self.send_encoded_to_engine(engine_id, request_type, payload, Vec::new()).await + } + + /// Send an add request, moving large tensor buffers into auxiliary frames. + pub async fn send_request_to_engine( + &self, + engine_id: &EngineId, + mut payload: EngineCoreRequest, + ) -> Result<()> { + let aux_frames = payload.extract_aux_frames(self.msgpack_zero_copy_threshold); + let payload = Bytes::from(encode_msgpack(&payload)?); + self.send_encoded_to_engine(engine_id, EngineCoreRequestType::Add, payload, aux_frames) + .await + } + + async fn send_encoded_to_engine( + &self, + engine_id: &EngineId, + request_type: EngineCoreRequestType, + payload: Bytes, + aux_frames: Vec, + ) -> Result<()> { let mut input_send = self.input_send.clone(); let engine_id = engine_id.clone(); @@ -242,6 +277,7 @@ impl ClientInner { &engine_id, request_type.to_frame(), payload, + aux_frames, ) .await }) diff --git a/rust/src/engine-core-client/src/protocol/logprobs.rs b/rust/src/engine-core-client/src/protocol/logprobs.rs index 6a52cab825..28f6327d9b 100644 --- a/rust/src/engine-core-client/src/protocol/logprobs.rs +++ b/rust/src/engine-core-client/src/protocol/logprobs.rs @@ -209,17 +209,17 @@ impl WireLogprobs { logprob_token_ids: WireNdArray { dtype: "( field: &str, frames: &[Frame], expected_scalars: &[ScalarType], -) -> Result<(Vec, Vec, ScalarType, Endianness)> +) -> Result<(Vec, Bytes, ScalarType, Endianness)> where Frame: AsRef<[u8]>, { @@ -171,7 +172,7 @@ pub(super) fn resolve_array_bytes( value: WireArrayData, field: &str, frames: &[Frame], -) -> Result> +) -> Result where Frame: AsRef<[u8]>, { @@ -187,7 +188,7 @@ where ), ) })?; - Ok(frame.as_ref().to_vec()) + Ok(Bytes::copy_from_slice(frame.as_ref())) } } } diff --git a/rust/src/engine-core-client/src/protocol/multimodal.rs b/rust/src/engine-core-client/src/protocol/multimodal.rs index 260812d00e..3a168d09e2 100644 --- a/rust/src/engine-core-client/src/protocol/multimodal.rs +++ b/rust/src/engine-core-client/src/protocol/multimodal.rs @@ -3,6 +3,7 @@ use std::collections::BTreeMap; +use bytes::Bytes; use serde::{Deserialize, Serialize}; use serde_tuple::{Deserialize_tuple, Serialize_tuple}; @@ -105,6 +106,44 @@ pub enum MmKwargValue { List(Vec), } +impl MmFeatureSpec { + /// Extract large tensor buffers from this feature in serialized field order. + pub(crate) fn extract_aux_frames(&mut self, aux_frames: &mut Vec, threshold: usize) { + if let Some(data) = &mut self.data { + for elem in data.values_mut() { + elem.extract_aux_frames(aux_frames, threshold); + } + } + if let Some(is_embed) = &mut self.mm_position.is_embed { + is_embed.extract_aux_frame(aux_frames, threshold); + } + } +} + +impl MmFieldElem { + /// Extract large tensor buffers from this field element. + fn extract_aux_frames(&mut self, aux_frames: &mut Vec, threshold: usize) { + if let Some(data) = &mut self.data { + data.extract_aux_frames(aux_frames, threshold); + } + } +} + +impl MmKwargValue { + /// Recursively extract large tensor buffers from this nested value. + fn extract_aux_frames(&mut self, aux_frames: &mut Vec, threshold: usize) { + match self { + Self::Tensor(tensor) => tensor.extract_aux_frame(aux_frames, threshold), + Self::List(values) => { + for value in values { + value.extract_aux_frames(aux_frames, threshold); + } + } + Self::Int(_) | Self::Float(_) => {} + } + } +} + /// Defines how to interpret tensor data belonging to a keyword argument for /// `MultiModalKwargsItems`, and vice versa. /// diff --git a/rust/src/engine-core-client/src/protocol/request.rs b/rust/src/engine-core-client/src/protocol/request.rs index ee1a6ac0b2..99e1b588e1 100644 --- a/rust/src/engine-core-client/src/protocol/request.rs +++ b/rust/src/engine-core-client/src/protocol/request.rs @@ -137,6 +137,17 @@ impl EngineCoreRequest { } Ok(()) } + + /// Extract large request tensors into ordered auxiliary frames. + pub(crate) fn extract_aux_frames(&mut self, threshold: usize) -> Vec { + let mut aux_frames = Vec::new(); + if let Some(features) = &mut self.mm_features { + for feature in features { + feature.extract_aux_frames(&mut aux_frames, threshold); + } + } + aux_frames + } } #[cfg(test)] @@ -144,9 +155,15 @@ mod tests { use rmpv::Value; use super::*; + use crate::protocol::multimodal::{ + MmBatchedField, MmFeatureSpec, MmField, MmFieldElem, MmKwargValue, PlaceholderRange, + }; use crate::protocol::sampling::EngineCoreSamplingParams; + use crate::protocol::tensor::{WireArrayData, WireTensor}; use crate::protocol::{decode_value, encode_msgpack}; + const AUX_FRAME_THRESHOLD: usize = 256; + #[test] fn engine_core_request_serializes_as_full_array() { let request = EngineCoreRequest { @@ -175,4 +192,84 @@ mod tests { assert_eq!(array[10], Value::Nil); assert_eq!(array[11], Value::from(7)); } + + #[test] + fn engine_core_request_extracts_large_nested_tensors_in_wire_order() { + let inline = vec![1_u8; AUX_FRAME_THRESHOLD - 1]; + let first_aux = vec![2_u8; AUX_FRAME_THRESHOLD]; + let second_aux = vec![3_u8; AUX_FRAME_THRESHOLD + 1]; + let first_aux_ptr = first_aux.as_ptr(); + let second_aux_ptr = second_aux.as_ptr(); + let mut request = EngineCoreRequest { + mm_features: Some(vec![MmFeatureSpec { + data: Some(BTreeMap::from([ + ( + "inline".to_string(), + MmFieldElem { + data: Some(MmKwargValue::Tensor(WireTensor::from_raw( + "uint8", + vec![inline.len()], + inline, + ))), + field: MmField::Batched(MmBatchedField { keep_on_cpu: false }), + }, + ), + ( + "nested".to_string(), + MmFieldElem { + data: Some(MmKwargValue::List(vec![ + MmKwargValue::Int(7), + MmKwargValue::Tensor(WireTensor::from_raw( + "uint8", + vec![first_aux.len()], + first_aux, + )), + ])), + field: MmField::Batched(MmBatchedField { keep_on_cpu: false }), + }, + ), + ])), + modality: "image".to_string(), + identifier: "id".to_string(), + mm_position: PlaceholderRange { + offset: 0, + length: second_aux.len(), + is_embed: Some(WireTensor::from_raw( + "bool", + vec![second_aux.len()], + second_aux, + )), + }, + mm_hash: None, + }]), + ..EngineCoreRequest::default() + }; + + let aux_frames = request.extract_aux_frames(AUX_FRAME_THRESHOLD); + + assert_eq!(aux_frames.len(), 2); + assert_eq!(aux_frames[0].as_ptr(), first_aux_ptr); + assert_eq!(aux_frames[1].as_ptr(), second_aux_ptr); + let feature = &request.mm_features.as_ref().unwrap()[0]; + let MmKwargValue::Tensor(inline) = + feature.data.as_ref().unwrap()["inline"].data.as_ref().unwrap() + else { + panic!("expected inline tensor"); + }; + assert!(matches!(inline.data, WireArrayData::RawView(_))); + let MmKwargValue::List(nested) = + feature.data.as_ref().unwrap()["nested"].data.as_ref().unwrap() + else { + panic!("expected nested tensor list"); + }; + let MmKwargValue::Tensor(nested_tensor) = &nested[1] else { + panic!("expected nested tensor"); + }; + assert_eq!(nested_tensor.data, WireArrayData::AuxIndex(1)); + assert_eq!( + feature.mm_position.is_embed.as_ref().unwrap().data, + WireArrayData::AuxIndex(2) + ); + assert!(request.extract_aux_frames(AUX_FRAME_THRESHOLD).is_empty()); + } } diff --git a/rust/src/engine-core-client/src/protocol/tensor.rs b/rust/src/engine-core-client/src/protocol/tensor.rs index 519bc238de..01540dab34 100644 --- a/rust/src/engine-core-client/src/protocol/tensor.rs +++ b/rust/src/engine-core-client/src/protocol/tensor.rs @@ -1,7 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project -use bytemuck::allocation::pod_collect_to_vec; +use bytemuck::{Pod, cast_slice}; +use bytes::Bytes; use enum_as_inner::EnumAsInner; use half::{bf16, f16}; use rmpv::Value; @@ -20,6 +21,21 @@ struct MsgpackExtRef<'a>((i8, ByteSlice<'a>)); struct ByteSlice<'a>(&'a [u8]); +struct PodVec(Vec); + +impl AsRef<[u8]> for PodVec { + fn as_ref(&self) -> &[u8] { + cast_slice(&self.0) + } +} + +fn bytes_from_pod_vec(data: Vec) -> Bytes +where + T: Pod + Send + 'static, +{ + Bytes::from_owner(PodVec(data)) +} + impl Serialize for ByteSlice<'_> { fn serialize(&self, serializer: S) -> std::result::Result where @@ -55,58 +71,63 @@ pub struct WireNdArray { impl WireNdArray { /// Build a float32 tensor/ndarray backed by native-endian raw-view bytes. - pub fn from_f32(shape: Vec, data: impl AsRef<[f32]>) -> Result { - let data = data.as_ref(); + /// + /// Takes ownership of the backing buffer without copying its data. + pub fn from_f32(shape: Vec, data: Vec) -> Result { validate_element_count(&shape, data.len())?; - Ok(Self { - dtype: "float32".to_string(), + Ok(Self::from_raw_bytes( + "float32", shape, - data: WireArrayData::RawView(pod_collect_to_vec::(data)), - }) + bytes_from_pod_vec(data), + )) } /// Build a float16 tensor/ndarray backed by native-endian raw-view bytes. - pub fn from_f16(shape: Vec, data: impl AsRef<[f16]>) -> Result { - let data = data.as_ref(); + /// + /// Takes ownership of the backing buffer without copying its data. + pub fn from_f16(shape: Vec, data: Vec) -> Result { validate_element_count(&shape, data.len())?; - Ok(Self { - dtype: "float16".to_string(), + Ok(Self::from_raw_bytes( + "float16", shape, - data: WireArrayData::RawView(pod_collect_to_vec::(data)), - }) + bytes_from_pod_vec(data), + )) } /// Build a bfloat16 tensor/ndarray backed by native-endian raw-view bytes. - pub fn from_bf16(shape: Vec, data: impl AsRef<[bf16]>) -> Result { - let data = data.as_ref(); + /// + /// Takes ownership of the backing buffer without copying its data. + pub fn from_bf16(shape: Vec, data: Vec) -> Result { validate_element_count(&shape, data.len())?; - Ok(Self { - dtype: "bfloat16".to_string(), + Ok(Self::from_raw_bytes( + "bfloat16", shape, - data: WireArrayData::RawView(pod_collect_to_vec::(data)), - }) + bytes_from_pod_vec(data), + )) } /// Build an int64 tensor/ndarray backed by native-endian raw-view bytes. - pub fn from_i64(shape: Vec, data: impl AsRef<[i64]>) -> Result { - let data = data.as_ref(); + /// + /// Takes ownership of the backing buffer without copying its data. + pub fn from_i64(shape: Vec, data: Vec) -> Result { validate_element_count(&shape, data.len())?; - Ok(Self { - dtype: "int64".to_string(), + Ok(Self::from_raw_bytes( + "int64", shape, - data: WireArrayData::RawView(pod_collect_to_vec::(data)), - }) + bytes_from_pod_vec(data), + )) } /// Build a uint32 tensor/ndarray backed by native-endian raw-view bytes. - pub fn from_u32(shape: Vec, data: impl AsRef<[u32]>) -> Result { - let data = data.as_ref(); + /// + /// Takes ownership of the backing buffer without copying its data. + pub fn from_u32(shape: Vec, data: Vec) -> Result { validate_element_count(&shape, data.len())?; - Ok(Self { - dtype: "uint32".to_string(), + Ok(Self::from_raw_bytes( + "uint32", shape, - data: WireArrayData::RawView(pod_collect_to_vec::(data)), - }) + bytes_from_pod_vec(data), + )) } /// Build a bool tensor/ndarray backed by raw-view bytes. @@ -118,7 +139,9 @@ impl WireNdArray { Ok(Self { dtype: "bool".to_string(), shape, - data: WireArrayData::RawView(data.iter().map(|value| u8::from(*value)).collect()), + data: WireArrayData::RawView(Bytes::from( + data.into_iter().map(u8::from).collect::>(), + )), }) } @@ -127,12 +150,22 @@ impl WireNdArray { /// Use this as an escape hatch when the caller already owns bytes that /// match the requested `dtype` and `shape`. pub fn from_raw(dtype: impl Into, shape: Vec, data: Vec) -> Self { + Self::from_raw_bytes(dtype, shape, Bytes::from(data)) + } + + /// Build a tensor/ndarray from an owned immutable raw-view buffer. + pub fn from_raw_bytes(dtype: impl Into, shape: Vec, data: Bytes) -> Self { Self { dtype: dtype.into(), shape, data: WireArrayData::RawView(data), } } + + /// Move a sufficiently large inline buffer into the ordered auxiliary-frame list. + pub(crate) fn extract_aux_frame(&mut self, aux_frames: &mut Vec, threshold: usize) { + self.data.extract_aux_frame(aux_frames, threshold); + } } /// Validate that the number of elements implied by the shape matches the length @@ -170,7 +203,25 @@ pub enum WireArrayData { /// stored. AuxIndex(usize), /// The raw bytes of this array/tensor. - RawView(Vec), + RawView(Bytes), +} + +impl WireArrayData { + /// Replace a sufficiently large raw view with its one-based auxiliary-frame index. + fn extract_aux_frame(&mut self, aux_frames: &mut Vec, threshold: usize) { + let Self::RawView(bytes) = self else { + return; + }; + if bytes.len() < threshold { + return; + } + + let index = aux_frames.len() + 1; + let bytes = std::mem::replace(self, Self::AuxIndex(index)) + .into_raw_view() + .expect("raw view was matched above"); + aux_frames.push(bytes); + } } impl<'de> Deserialize<'de> for WireArrayData { @@ -180,7 +231,9 @@ impl<'de> Deserialize<'de> for WireArrayData { { let value = Value::deserialize(deserializer)?; match value { - Value::Ext(tag, bytes) if tag == CUSTOM_TYPE_RAW_VIEW => Ok(Self::RawView(bytes)), + Value::Ext(tag, bytes) if tag == CUSTOM_TYPE_RAW_VIEW => { + Ok(Self::RawView(Bytes::from(bytes))) + } Value::Ext(tag, _) => Err(serde::de::Error::custom(format!( "unsupported extension type code {tag}" ))), @@ -201,9 +254,6 @@ impl Serialize for WireArrayData { where S: Serializer, { - // TODO: outbound request serialization currently only supports inline - // raw-view bytes. Emitting aux frames needs transport-level plumbing; - // serializing `AuxIndex` here only preserves an already-built reference. match self { Self::AuxIndex(index) => serializer.serialize_u64(*index as u64), Self::RawView(bytes) => { @@ -221,7 +271,7 @@ mod tests { fn raw_view_serializes_as_msgpack_ext() { let bytes = vec![1, 2, 3, 4]; let encoded = - rmp_serde::to_vec_named(&WireArrayData::RawView(bytes.clone())).expect("encode"); + rmp_serde::to_vec_named(&WireArrayData::RawView(bytes.clone().into())).expect("encode"); let expected = rmp_serde::to_vec_named(&Value::Ext(CUSTOM_TYPE_RAW_VIEW, bytes.clone())) .expect("encode expected"); @@ -234,11 +284,15 @@ mod tests { #[test] fn constructors_build_raw_view_tensors() { - let f32_tensor = WireNdArray::from_f32(vec![2], vec![1.0, 2.5]).unwrap(); + let f32_data = vec![1.0, 2.5]; + let f32_data_ptr = f32_data.as_ptr().cast::(); + let f32_tensor = WireNdArray::from_f32(vec![2], f32_data).unwrap(); assert_eq!(f32_tensor.dtype, "float32"); assert_eq!(f32_tensor.shape, vec![2]); + let f32_raw_view = f32_tensor.data.into_raw_view().expect("raw view"); + assert_eq!(f32_raw_view.as_ptr(), f32_data_ptr); assert_eq!( - f32_tensor.data.into_raw_view().expect("raw view"), + f32_raw_view, [1.0_f32, 2.5].into_iter().flat_map(f32::to_ne_bytes).collect::>() ); @@ -258,15 +312,15 @@ mod tests { let i64_tensor = WireNdArray::from_i64(vec![1], vec![-7]).unwrap(); assert_eq!(i64_tensor.dtype, "int64"); assert_eq!( - i64_tensor.data.into_raw_view().expect("raw view"), - (-7_i64).to_ne_bytes() + i64_tensor.data.into_raw_view().expect("raw view").as_ref(), + (-7_i64).to_ne_bytes().as_ref() ); let u32_tensor = WireNdArray::from_u32(vec![1], vec![42]).unwrap(); assert_eq!(u32_tensor.dtype, "uint32"); assert_eq!( - u32_tensor.data.into_raw_view().expect("raw view"), - 42_u32.to_ne_bytes() + u32_tensor.data.into_raw_view().expect("raw view").as_ref(), + 42_u32.to_ne_bytes().as_ref() ); let bool_tensor = WireNdArray::from_bool(vec![2], vec![false, true]).unwrap(); diff --git a/rust/src/engine-core-client/src/tests/client.rs b/rust/src/engine-core-client/src/tests/client.rs index 64b80185c3..7b3e688d86 100644 --- a/rust/src/engine-core-client/src/tests/client.rs +++ b/rust/src/engine-core-client/src/tests/client.rs @@ -32,7 +32,7 @@ use crate::protocol::output::{ use crate::protocol::request::{EngineCoreRequest, EngineCoreRequestType}; use crate::protocol::sampling::EngineCoreSamplingParams; use crate::protocol::stats::SchedulerStats; -use crate::protocol::tensor::WireTensor; +use crate::protocol::tensor::{WireArrayData, WireTensor}; use crate::protocol::utility::{UtilityOutput, UtilityResultEnvelope}; use crate::test_utils::{ IpcNamespace, setup_bootstrapped_mock_engine, setup_mock_engine_sockets, @@ -1728,6 +1728,86 @@ async fn client_decodes_multipart_logprob_outputs() { client.shutdown().await.unwrap(); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn client_sends_large_multimodal_tensor_as_aux_frame() { + init_tracing(); + let ipc = IpcNamespace::new().unwrap(); + let handshake_address = ipc.handshake_endpoint(); + let engine_id = b"engine-multimodal-aux".to_vec(); + let tensor_data = (0..64).map(|value| value as f32).collect::>(); + let expected_bytes = + tensor_data.iter().flat_map(|value| value.to_ne_bytes()).collect::>(); + let mut request = sample_multimodal_request(); + request.mm_features.as_mut().unwrap()[0] + .data + .as_mut() + .unwrap() + .get_mut("pixel_values") + .unwrap() + .data = Some(MmKwargValue::Tensor( + WireTensor::from_f32(vec![64], tensor_data).unwrap(), + )); + + let (shutdown_tx, engine_task) = spawn_mock_engine_task( + handshake_address.clone(), + engine_id.clone(), + move |dealer, push| { + Box::pin(async move { + let add = recv_engine_message(dealer).await; + assert_eq!(add.len(), 3); + assert_eq!(add[0].as_ref(), &[0x00]); + let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap(); + let MmKwargValue::Tensor(tensor) = + request.mm_features.as_ref().unwrap()[0].data.as_ref().unwrap()["pixel_values"] + .data + .as_ref() + .unwrap() + else { + panic!("expected tensor"); + }; + assert_eq!(tensor.data, WireArrayData::AuxIndex(1)); + assert_eq!(add[2].as_ref(), expected_bytes); + + send_outputs( + push, + RequestBatchOutputs { + outputs: vec![request_output( + "req-mm", + vec![], + Some(EngineCoreFinishReason::Length), + )], + finished_requests: Some(BTreeSet::from(["req-mm".to_string()])), + ..Default::default() + } + .into(), + ) + .await; + }) + }, + ); + + let client = connect_client_with_ipc( + handshake_test_config( + handshake_address, + 1, + "test-model", + Duration::from_secs(2), + 0, + None, + ), + &ipc, + ) + .await; + + let outputs = client.call(request).await.unwrap().collect::>().await; + assert_eq!(outputs.len(), 1); + assert!(outputs[0].is_ok()); + + let _ = shutdown_tx.send(()); + engine_task.await.unwrap(); + client.shutdown().await.unwrap(); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn multi_engine_client_shares_transport_and_routes_by_inflight_count() { init_tracing(); diff --git a/rust/src/engine-core-client/src/transport.rs b/rust/src/engine-core-client/src/transport.rs index d2eb57aa14..d3e3ecae2b 100644 --- a/rust/src/engine-core-client/src/transport.rs +++ b/rust/src/engine-core-client/src/transport.rs @@ -512,14 +512,14 @@ pub async fn send_message( input_send: &mut RouterSendHalf, engine_id: &EngineId, request_type: Bytes, - payload: Vec, + payload: Bytes, + aux_frames: Vec, ) -> Result<()> { - let message = ZmqMessage::try_from(vec![ - engine_id.to_frame(), - request_type, - Bytes::from(payload), - ]) - .expect("router messages must contain identity and payload"); + let mut frames = Vec::with_capacity(3 + aux_frames.len()); + frames.extend([engine_id.to_frame(), request_type, payload]); + frames.extend(aux_frames); + let message = + ZmqMessage::try_from(frames).expect("router messages must contain identity and payload"); trace!( ?engine_id,