[Rust Frontend] Send multimodal tensors in auxiliary frames (#49341)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: reidliu41 <reid201711@gmail.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -222,7 +222,9 @@ mod tests {
|
||||
assert_eq!(tensor.shape, vec![expected.len()]);
|
||||
assert_eq!(
|
||||
tensor.data,
|
||||
WireArrayData::RawView(expected.iter().map(|value| u8::from(*value)).collect())
|
||||
WireArrayData::RawView(
|
||||
expected.iter().map(|value| u8::from(*value)).collect::<Vec<_>>().into(),
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -2,28 +2,58 @@
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::mem::size_of;
|
||||
|
||||
use half::{bf16, f16};
|
||||
use llm_multimodal::{ModelSpecificValue, PreprocessedEncoderInputs};
|
||||
use vllm_engine_core_client::protocol::dtype::ModelDtype;
|
||||
use vllm_engine_core_client::protocol::multimodal::MmKwargValue as ProtocolKwargValue;
|
||||
use vllm_engine_core_client::protocol::tensor::{ShapeExt as _, WireTensor};
|
||||
use vllm_engine_core_client::protocol::tensor::{ShapeExt as _, WireArrayData, WireTensor};
|
||||
|
||||
use crate::error::{Error, Result, bail_multimodal, multimodal};
|
||||
|
||||
/// Element type retained alongside an encoded tensor during multimodal lowering.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub(super) enum TensorKind {
|
||||
/// 32-bit floating point.
|
||||
F32,
|
||||
/// IEEE 16-bit floating point.
|
||||
F16,
|
||||
/// Brain floating point.
|
||||
Bf16,
|
||||
/// Signed 64-bit integer.
|
||||
I64,
|
||||
/// Unsigned 32-bit integer.
|
||||
U32,
|
||||
}
|
||||
|
||||
impl TensorKind {
|
||||
const fn element_size(self) -> usize {
|
||||
match self {
|
||||
Self::F32 => size_of::<f32>(),
|
||||
Self::F16 => size_of::<f16>(),
|
||||
Self::Bf16 => size_of::<bf16>(),
|
||||
Self::I64 => size_of::<i64>(),
|
||||
Self::U32 => size_of::<u32>(),
|
||||
}
|
||||
}
|
||||
|
||||
const fn wire_dtype(self) -> &'static str {
|
||||
match self {
|
||||
Self::F32 => "float32",
|
||||
Self::F16 => "float16",
|
||||
Self::Bf16 => "bfloat16",
|
||||
Self::I64 => "int64",
|
||||
Self::U32 => "uint32",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Representation for multimodal kwarg values for transformation.
|
||||
#[derive(Debug)]
|
||||
pub(super) enum KwargValue {
|
||||
/// Float tensor with row-major flat data and shape.
|
||||
F32Tensor { data: Vec<f32>, shape: Vec<usize> },
|
||||
/// Float16 tensor with row-major flat data and shape.
|
||||
F16Tensor { data: Vec<f16>, shape: Vec<usize> },
|
||||
/// BFloat16 tensor with row-major flat data and shape.
|
||||
Bf16Tensor { data: Vec<bf16>, shape: Vec<usize> },
|
||||
/// Signed integer tensor with row-major flat data and shape.
|
||||
I64Tensor { data: Vec<i64>, shape: Vec<usize> },
|
||||
/// Unsigned integer tensor with row-major flat data and shape.
|
||||
U32Tensor { data: Vec<u32>, shape: Vec<usize> },
|
||||
/// Tensor with row-major flat data and shape.
|
||||
Tensor { kind: TensorKind, wire: WireTensor },
|
||||
/// Non-tensor kwarg value that is shared or copied as-is.
|
||||
Passthrough(ProtocolKwargValue),
|
||||
}
|
||||
@@ -67,8 +97,14 @@ impl KwargValue {
|
||||
ModelSpecificValue::Tensor { data, shape } => {
|
||||
Self::from_f32_tensor(data, shape, float_dtype)?
|
||||
}
|
||||
ModelSpecificValue::IntTensor { data, shape } => Self::I64Tensor { data, shape },
|
||||
ModelSpecificValue::UintTensor { data, shape } => Self::U32Tensor { data, shape },
|
||||
ModelSpecificValue::IntTensor { data, shape } => {
|
||||
let wire = WireTensor::from_i64(shape, data).map_err(Error::Multimodal)?;
|
||||
Self::tensor(TensorKind::I64, wire)
|
||||
}
|
||||
ModelSpecificValue::UintTensor { data, shape } => {
|
||||
let wire = WireTensor::from_u32(shape, data).map_err(Error::Multimodal)?;
|
||||
Self::tensor(TensorKind::U32, wire)
|
||||
}
|
||||
ModelSpecificValue::Int(value) => Self::Passthrough(Int(value)),
|
||||
ModelSpecificValue::Float(value) => Self::Passthrough(Float(value)),
|
||||
ModelSpecificValue::IntVec(values) => {
|
||||
@@ -93,17 +129,23 @@ impl KwargValue {
|
||||
/// Convert a float tensor to the target float dtype if needed, keeping the
|
||||
/// same shape.
|
||||
fn from_f32_tensor(data: Vec<f32>, shape: Vec<usize>, float_dtype: ModelDtype) -> Result<Self> {
|
||||
match float_dtype {
|
||||
ModelDtype::Float16 => Ok(Self::F16Tensor {
|
||||
data: data.into_iter().map(f16::from_f32).collect(),
|
||||
shape,
|
||||
}),
|
||||
ModelDtype::BFloat16 => Ok(Self::Bf16Tensor {
|
||||
data: data.into_iter().map(bf16::from_f32).collect(),
|
||||
shape,
|
||||
}),
|
||||
ModelDtype::Float32 => Ok(Self::F32Tensor { data, shape }),
|
||||
}
|
||||
let (kind, wire) = match float_dtype {
|
||||
ModelDtype::Float16 => (
|
||||
TensorKind::F16,
|
||||
WireTensor::from_f16(shape, data.into_iter().map(f16::from_f32).collect()),
|
||||
),
|
||||
ModelDtype::BFloat16 => (
|
||||
TensorKind::Bf16,
|
||||
WireTensor::from_bf16(shape, data.into_iter().map(bf16::from_f32).collect()),
|
||||
),
|
||||
ModelDtype::Float32 => (TensorKind::F32, WireTensor::from_f32(shape, data)),
|
||||
};
|
||||
wire.map(|wire| Self::tensor(kind, wire)).map_err(Error::Multimodal)
|
||||
}
|
||||
|
||||
fn tensor(kind: TensorKind, wire: WireTensor) -> Self {
|
||||
debug_assert_eq!(wire.dtype, kind.wire_dtype());
|
||||
Self::Tensor { kind, wire }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -111,15 +153,11 @@ impl TryFrom<&KwargValue> for ProtocolKwargValue {
|
||||
type Error = Error;
|
||||
|
||||
fn try_from(value: &KwargValue) -> Result<Self> {
|
||||
let tensor = match value {
|
||||
KwargValue::F32Tensor { data, shape } => WireTensor::from_f32(shape.clone(), data),
|
||||
KwargValue::F16Tensor { data, shape } => WireTensor::from_f16(shape.clone(), data),
|
||||
KwargValue::Bf16Tensor { data, shape } => WireTensor::from_bf16(shape.clone(), data),
|
||||
KwargValue::I64Tensor { data, shape } => WireTensor::from_i64(shape.clone(), data),
|
||||
KwargValue::U32Tensor { data, shape } => WireTensor::from_u32(shape.clone(), data),
|
||||
let wire = match value {
|
||||
KwargValue::Tensor { wire, .. } => wire.clone(),
|
||||
KwargValue::Passthrough(value) => return Ok(value.clone()),
|
||||
};
|
||||
tensor.map(ProtocolKwargValue::Tensor).map_err(Error::Multimodal)
|
||||
Ok(ProtocolKwargValue::Tensor(wire))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -127,11 +165,7 @@ impl KwargValue {
|
||||
/// First-axis length for tensor values; `None` for passthrough kwargs.
|
||||
pub(super) fn first_dim(&self) -> Option<usize> {
|
||||
match self {
|
||||
Self::F32Tensor { shape, .. }
|
||||
| Self::F16Tensor { shape, .. }
|
||||
| Self::Bf16Tensor { shape, .. }
|
||||
| Self::I64Tensor { shape, .. }
|
||||
| Self::U32Tensor { shape, .. } => shape.first().copied(),
|
||||
Self::Tensor { wire, .. } => wire.shape.first().copied(),
|
||||
Self::Passthrough(_) => None,
|
||||
}
|
||||
}
|
||||
@@ -161,30 +195,13 @@ impl KwargValue {
|
||||
end: usize,
|
||||
drop_axis: bool,
|
||||
) -> Result<ProtocolKwargValue> {
|
||||
let tensor = match self {
|
||||
Self::F32Tensor { data, shape } => {
|
||||
let (shape, data) = slice_first_axis_range(shape, data, start, end, drop_axis)?;
|
||||
WireTensor::from_f32(shape, data)
|
||||
}
|
||||
Self::F16Tensor { data, shape } => {
|
||||
let (shape, data) = slice_first_axis_range(shape, data, start, end, drop_axis)?;
|
||||
WireTensor::from_f16(shape, data)
|
||||
}
|
||||
Self::Bf16Tensor { data, shape } => {
|
||||
let (shape, data) = slice_first_axis_range(shape, data, start, end, drop_axis)?;
|
||||
WireTensor::from_bf16(shape, data)
|
||||
}
|
||||
Self::I64Tensor { data, shape } => {
|
||||
let (shape, data) = slice_first_axis_range(shape, data, start, end, drop_axis)?;
|
||||
WireTensor::from_i64(shape, data)
|
||||
}
|
||||
Self::U32Tensor { data, shape } => {
|
||||
let (shape, data) = slice_first_axis_range(shape, data, start, end, drop_axis)?;
|
||||
WireTensor::from_u32(shape, data)
|
||||
let wire = match self {
|
||||
Self::Tensor { kind, wire } => {
|
||||
slice_first_axis_range(wire, kind.element_size(), start, end, drop_axis)
|
||||
}
|
||||
Self::Passthrough(value) => return Ok(value.clone()),
|
||||
};
|
||||
tensor.map(ProtocolKwargValue::Tensor).map_err(Error::Multimodal)
|
||||
wire.map(ProtocolKwargValue::Tensor)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -208,44 +225,53 @@ pub(super) fn flat_range_for_index(
|
||||
/// Read a tensor value as per-image sizes for flat slicing.
|
||||
fn tensor_as_usize_vec(tensor: &KwargValue) -> Result<Vec<usize>> {
|
||||
match tensor {
|
||||
KwargValue::I64Tensor { data, .. } => data
|
||||
.iter()
|
||||
KwargValue::Tensor {
|
||||
kind: TensorKind::I64,
|
||||
wire,
|
||||
} => raw_tensor_bytes(wire, size_of::<i64>())?
|
||||
.chunks_exact(size_of::<i64>())
|
||||
.map(|bytes| i64::from_ne_bytes(bytes.try_into().expect("exact int64 chunk")))
|
||||
.map(|value| {
|
||||
usize::try_from(*value)
|
||||
usize::try_from(value)
|
||||
.map_err(|_| multimodal!("negative flat tensor size `{value}`"))
|
||||
})
|
||||
.collect(),
|
||||
KwargValue::U32Tensor { data, .. } => {
|
||||
Ok(data.iter().map(|value| *value as usize).collect())
|
||||
}
|
||||
KwargValue::Tensor {
|
||||
kind: TensorKind::U32,
|
||||
wire,
|
||||
} => Ok(raw_tensor_bytes(wire, size_of::<u32>())?
|
||||
.chunks_exact(size_of::<u32>())
|
||||
.map(|bytes| u32::from_ne_bytes(bytes.try_into().expect("exact uint32 chunk")) as usize)
|
||||
.collect()),
|
||||
_ => Err(multimodal!("flat tensor sizes must be int64 or uint32")),
|
||||
}
|
||||
}
|
||||
|
||||
/// Slice a flat row-major tensor along its first axis.
|
||||
fn slice_first_axis_range<'a, T>(
|
||||
shape: &[usize],
|
||||
data: &'a [T],
|
||||
fn slice_first_axis_range(
|
||||
tensor: &WireTensor,
|
||||
element_size: usize,
|
||||
start: usize,
|
||||
end: usize,
|
||||
drop_axis: bool,
|
||||
) -> Result<(Vec<usize>, &'a [T])> {
|
||||
) -> Result<WireTensor> {
|
||||
let shape = tensor.shape.as_slice();
|
||||
raw_tensor_bytes(tensor, element_size)?;
|
||||
let first_dim = *shape.first().ok_or_else(|| multimodal!("tensor has no first dimension"))?;
|
||||
if start > end || end > first_dim {
|
||||
bail_multimodal!("invalid tensor slice {start}..{end} for first dimension {first_dim}");
|
||||
}
|
||||
let expected_len = shape
|
||||
.checked_numel()
|
||||
.ok_or_else(|| multimodal!("tensor shape {shape:?} has too many elements"))?;
|
||||
if expected_len != data.len() {
|
||||
bail_multimodal!(
|
||||
"tensor shape {shape:?} expects {expected_len} elements, got {}",
|
||||
data.len()
|
||||
);
|
||||
}
|
||||
let stride = shape[1..].iter().product::<usize>();
|
||||
let data_start = start * stride;
|
||||
let data_end = end * stride;
|
||||
let stride = shape[1..]
|
||||
.iter()
|
||||
.try_fold(1usize, |acc, dim| acc.checked_mul(*dim))
|
||||
.and_then(|stride| stride.checked_mul(element_size))
|
||||
.ok_or_else(|| multimodal!("tensor shape {shape:?} byte stride overflowed usize"))?;
|
||||
let data_start = start
|
||||
.checked_mul(stride)
|
||||
.ok_or_else(|| multimodal!("tensor slice start byte offset overflowed usize"))?;
|
||||
let data_end = end
|
||||
.checked_mul(stride)
|
||||
.ok_or_else(|| multimodal!("tensor slice end byte offset overflowed usize"))?;
|
||||
let out_shape = if drop_axis {
|
||||
shape[1..].to_vec()
|
||||
} else {
|
||||
@@ -253,7 +279,38 @@ fn slice_first_axis_range<'a, T>(
|
||||
shape[0] = end - start;
|
||||
shape
|
||||
};
|
||||
Ok((out_shape, &data[data_start..data_end]))
|
||||
let WireArrayData::RawView(data) = &tensor.data else {
|
||||
return Err(multimodal!("cannot slice an aux tensor buffer"));
|
||||
};
|
||||
Ok(WireTensor::from_raw_bytes(
|
||||
tensor.dtype.clone(),
|
||||
out_shape,
|
||||
data.slice(data_start..data_end),
|
||||
))
|
||||
}
|
||||
|
||||
fn raw_tensor_bytes(tensor: &WireTensor, element_size: usize) -> Result<&[u8]> {
|
||||
let WireArrayData::RawView(data) = &tensor.data else {
|
||||
return Err(multimodal!("expected an inline tensor buffer"));
|
||||
};
|
||||
let expected_bytes = tensor
|
||||
.shape
|
||||
.checked_numel()
|
||||
.and_then(|numel| numel.checked_mul(element_size))
|
||||
.ok_or_else(|| {
|
||||
multimodal!(
|
||||
"tensor shape {:?} byte length overflowed usize",
|
||||
tensor.shape
|
||||
)
|
||||
})?;
|
||||
if expected_bytes != data.len() {
|
||||
bail_multimodal!(
|
||||
"tensor shape {:?} expects {expected_bytes} bytes, got {}",
|
||||
tensor.shape,
|
||||
data.len()
|
||||
);
|
||||
}
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -262,28 +319,32 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn batched_wire_value_at_drops_first_axis() {
|
||||
let value = KwargValue::F32Tensor {
|
||||
data: vec![1.0, 2.0, 3.0, 4.0],
|
||||
shape: vec![2, 2],
|
||||
};
|
||||
let data = vec![1.0_f32, 2.0, 3.0, 4.0];
|
||||
let expected_ptr = data.as_ptr().cast::<u8>().wrapping_add(2 * size_of::<f32>());
|
||||
let value = KwargValue::tensor(
|
||||
TensorKind::F32,
|
||||
WireTensor::from_f32(vec![2, 2], data).unwrap(),
|
||||
);
|
||||
|
||||
let ProtocolKwargValue::Tensor(tensor) = value.batched_wire_value_at(1).unwrap() else {
|
||||
panic!("expected tensor");
|
||||
};
|
||||
|
||||
assert_eq!(tensor.shape, vec![2]);
|
||||
let raw_view = tensor.data.into_raw_view().unwrap();
|
||||
assert_eq!(raw_view.as_ptr(), expected_ptr);
|
||||
assert_eq!(
|
||||
tensor.data.into_raw_view().unwrap(),
|
||||
raw_view,
|
||||
[3.0_f32, 4.0].into_iter().flat_map(f32::to_ne_bytes).collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flat_wire_value_range_keeps_first_axis() {
|
||||
let value = KwargValue::U32Tensor {
|
||||
data: (0..10).collect(),
|
||||
shape: vec![5, 2],
|
||||
};
|
||||
let value = KwargValue::tensor(
|
||||
TensorKind::U32,
|
||||
WireTensor::from_u32(vec![5, 2], (0..10_u32).collect()).unwrap(),
|
||||
);
|
||||
|
||||
let ProtocolKwargValue::Tensor(tensor) = value.flat_wire_value_range(1, 3).unwrap() else {
|
||||
panic!("expected tensor");
|
||||
@@ -298,10 +359,10 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn flat_range_for_index_uses_size_tensor() {
|
||||
let sizes = KwargValue::I64Tensor {
|
||||
data: vec![2, 3, 4],
|
||||
shape: vec![3],
|
||||
};
|
||||
let sizes = KwargValue::tensor(
|
||||
TensorKind::I64,
|
||||
WireTensor::from_i64(vec![3], vec![2_i64, 3, 4]).unwrap(),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
flat_range_for_index(&sizes, "image_grid_thw", 1).unwrap(),
|
||||
@@ -311,10 +372,11 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn slice_first_axis_range_errors_on_shape_data_mismatch() {
|
||||
let error = slice_first_axis_range(&[2, 2], &[1.0_f32, 2.0, 3.0], 0, 1, true).unwrap_err();
|
||||
let tensor = WireTensor::from_raw("float32", vec![2, 2], vec![0; 3 * size_of::<f32>()]);
|
||||
let error = slice_first_axis_range(&tensor, size_of::<f32>(), 0, 1, true).unwrap_err();
|
||||
|
||||
assert!(
|
||||
matches!(error, Error::Multimodal(message) if message.contains("expects 4 elements"))
|
||||
matches!(error, Error::Multimodal(message) if message.contains("expects 16 bytes"))
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -519,7 +519,7 @@ impl EngineCoreClient {
|
||||
"registered request to engine"
|
||||
);
|
||||
|
||||
self.inner.send_to_engine(&engine_id, EngineCoreRequestType::Add, &req).await?;
|
||||
self.inner.send_request_to_engine(&engine_id, req).await?;
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
|
||||
@@ -6,6 +6,7 @@ use std::sync::Arc;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use arc_swap::ArcSwapOption;
|
||||
use bytes::Bytes;
|
||||
use parking_lot::Mutex;
|
||||
use thiserror_ext::AsReport as _;
|
||||
use tokio::runtime::Handle;
|
||||
@@ -21,17 +22,30 @@ use crate::error::{client_closed, dispatcher_closed, unexpected_dispatcher_outpu
|
||||
use crate::metrics::{LoraInfoExporter, SchedulerStatsRecorder};
|
||||
use crate::protocol::encode_msgpack;
|
||||
use crate::protocol::output::{EngineCoreOutput, EngineCoreOutputs};
|
||||
use crate::protocol::request::EngineCoreRequestType;
|
||||
use crate::protocol::request::{EngineCoreRequest, EngineCoreRequestType};
|
||||
use crate::protocol::stats::SchedulerStats;
|
||||
use crate::protocol::utility::UtilityOutput;
|
||||
use crate::transport::{ConnectedEngine, EngineId};
|
||||
use crate::{Error, Result, transport};
|
||||
|
||||
const MSGPACK_ZERO_COPY_THRESHOLD_ENV: &str = "VLLM_MSGPACK_ZERO_COPY_THRESHOLD";
|
||||
const DEFAULT_MSGPACK_ZERO_COPY_THRESHOLD: usize = 256;
|
||||
|
||||
fn msgpack_zero_copy_threshold() -> usize {
|
||||
std::env::var(MSGPACK_ZERO_COPY_THRESHOLD_ENV)
|
||||
.ok()
|
||||
.and_then(|value| value.parse().ok())
|
||||
.unwrap_or(DEFAULT_MSGPACK_ZERO_COPY_THRESHOLD)
|
||||
}
|
||||
|
||||
pub(crate) struct ClientInner {
|
||||
input_send: RouterSendHalf,
|
||||
/// The runtime handle used for sending messages to the engine.
|
||||
handle: Handle,
|
||||
model_name: String,
|
||||
/// Per-tensor byte threshold loaded from env variable
|
||||
/// `VLLM_MSGPACK_ZERO_COPY_THRESHOLD` when this inner client is created.
|
||||
msgpack_zero_copy_threshold: usize,
|
||||
scheduler_stats_recorder: SchedulerStatsRecorder,
|
||||
request_reg: Mutex<RequestRegistry>,
|
||||
utility_reg: Mutex<UtilityRegistry>,
|
||||
@@ -54,6 +68,7 @@ impl ClientInner {
|
||||
input_send,
|
||||
handle,
|
||||
model_name,
|
||||
msgpack_zero_copy_threshold: msgpack_zero_copy_threshold(),
|
||||
scheduler_stats_recorder,
|
||||
request_reg: Mutex::new(RequestRegistry::new(engines)),
|
||||
utility_reg: Mutex::new(UtilityRegistry::default()),
|
||||
@@ -229,9 +244,29 @@ impl ClientInner {
|
||||
where
|
||||
T: serde::Serialize + std::fmt::Debug,
|
||||
{
|
||||
// TODO: for `EngineCoreRequest`, split outbound tensor raw views into aux
|
||||
// frames instead of always producing a single msgpack frame.
|
||||
let payload = encode_msgpack(payload)?;
|
||||
let payload = Bytes::from(encode_msgpack(payload)?);
|
||||
self.send_encoded_to_engine(engine_id, request_type, payload, Vec::new()).await
|
||||
}
|
||||
|
||||
/// Send an add request, moving large tensor buffers into auxiliary frames.
|
||||
pub async fn send_request_to_engine(
|
||||
&self,
|
||||
engine_id: &EngineId,
|
||||
mut payload: EngineCoreRequest,
|
||||
) -> Result<()> {
|
||||
let aux_frames = payload.extract_aux_frames(self.msgpack_zero_copy_threshold);
|
||||
let payload = Bytes::from(encode_msgpack(&payload)?);
|
||||
self.send_encoded_to_engine(engine_id, EngineCoreRequestType::Add, payload, aux_frames)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn send_encoded_to_engine(
|
||||
&self,
|
||||
engine_id: &EngineId,
|
||||
request_type: EngineCoreRequestType,
|
||||
payload: Bytes,
|
||||
aux_frames: Vec<Bytes>,
|
||||
) -> Result<()> {
|
||||
let mut input_send = self.input_send.clone();
|
||||
let engine_id = engine_id.clone();
|
||||
|
||||
@@ -242,6 +277,7 @@ impl ClientInner {
|
||||
&engine_id,
|
||||
request_type.to_frame(),
|
||||
payload,
|
||||
aux_frames,
|
||||
)
|
||||
.await
|
||||
})
|
||||
|
||||
@@ -209,17 +209,17 @@ impl WireLogprobs {
|
||||
logprob_token_ids: WireNdArray {
|
||||
dtype: "<i8".to_string(),
|
||||
shape: vec![rows, cols],
|
||||
data: WireArrayData::RawView(token_ids),
|
||||
data: WireArrayData::RawView(token_ids.into()),
|
||||
},
|
||||
logprobs: WireNdArray {
|
||||
dtype: "<f4".to_string(),
|
||||
shape: vec![rows, cols],
|
||||
data: WireArrayData::RawView(logprobs),
|
||||
data: WireArrayData::RawView(logprobs.into()),
|
||||
},
|
||||
token_ranks: WireNdArray {
|
||||
dtype: "<i8".to_string(),
|
||||
shape: vec![rows],
|
||||
data: WireArrayData::RawView(token_ranks),
|
||||
data: WireArrayData::RawView(token_ranks.into()),
|
||||
},
|
||||
cu_num_generated_tokens: None,
|
||||
})
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
use std::io::Cursor;
|
||||
|
||||
use byteorder::{BigEndian, LittleEndian, NativeEndian, ReadBytesExt};
|
||||
use bytes::Bytes;
|
||||
use itertools::Itertools as _;
|
||||
|
||||
use crate::error::{Error, Result, ext_value_decode};
|
||||
@@ -126,7 +127,7 @@ pub(super) fn decode_array_metadata<Frame>(
|
||||
field: &str,
|
||||
frames: &[Frame],
|
||||
expected_scalars: &[ScalarType],
|
||||
) -> Result<(Vec<usize>, Vec<u8>, ScalarType, Endianness)>
|
||||
) -> Result<(Vec<usize>, Bytes, ScalarType, Endianness)>
|
||||
where
|
||||
Frame: AsRef<[u8]>,
|
||||
{
|
||||
@@ -171,7 +172,7 @@ pub(super) fn resolve_array_bytes<Frame>(
|
||||
value: WireArrayData,
|
||||
field: &str,
|
||||
frames: &[Frame],
|
||||
) -> Result<Vec<u8>>
|
||||
) -> Result<Bytes>
|
||||
where
|
||||
Frame: AsRef<[u8]>,
|
||||
{
|
||||
@@ -187,7 +188,7 @@ where
|
||||
),
|
||||
)
|
||||
})?;
|
||||
Ok(frame.as_ref().to_vec())
|
||||
Ok(Bytes::copy_from_slice(frame.as_ref()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use bytes::Bytes;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
|
||||
|
||||
@@ -105,6 +106,44 @@ pub enum MmKwargValue {
|
||||
List(Vec<MmKwargValue>),
|
||||
}
|
||||
|
||||
impl MmFeatureSpec {
|
||||
/// Extract large tensor buffers from this feature in serialized field order.
|
||||
pub(crate) fn extract_aux_frames(&mut self, aux_frames: &mut Vec<Bytes>, threshold: usize) {
|
||||
if let Some(data) = &mut self.data {
|
||||
for elem in data.values_mut() {
|
||||
elem.extract_aux_frames(aux_frames, threshold);
|
||||
}
|
||||
}
|
||||
if let Some(is_embed) = &mut self.mm_position.is_embed {
|
||||
is_embed.extract_aux_frame(aux_frames, threshold);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl MmFieldElem {
|
||||
/// Extract large tensor buffers from this field element.
|
||||
fn extract_aux_frames(&mut self, aux_frames: &mut Vec<Bytes>, threshold: usize) {
|
||||
if let Some(data) = &mut self.data {
|
||||
data.extract_aux_frames(aux_frames, threshold);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl MmKwargValue {
|
||||
/// Recursively extract large tensor buffers from this nested value.
|
||||
fn extract_aux_frames(&mut self, aux_frames: &mut Vec<Bytes>, threshold: usize) {
|
||||
match self {
|
||||
Self::Tensor(tensor) => tensor.extract_aux_frame(aux_frames, threshold),
|
||||
Self::List(values) => {
|
||||
for value in values {
|
||||
value.extract_aux_frames(aux_frames, threshold);
|
||||
}
|
||||
}
|
||||
Self::Int(_) | Self::Float(_) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Defines how to interpret tensor data belonging to a keyword argument for
|
||||
/// `MultiModalKwargsItems`, and vice versa.
|
||||
///
|
||||
|
||||
@@ -137,6 +137,17 @@ impl EngineCoreRequest {
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Extract large request tensors into ordered auxiliary frames.
|
||||
pub(crate) fn extract_aux_frames(&mut self, threshold: usize) -> Vec<Bytes> {
|
||||
let mut aux_frames = Vec::new();
|
||||
if let Some(features) = &mut self.mm_features {
|
||||
for feature in features {
|
||||
feature.extract_aux_frames(&mut aux_frames, threshold);
|
||||
}
|
||||
}
|
||||
aux_frames
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -144,9 +155,15 @@ mod tests {
|
||||
use rmpv::Value;
|
||||
|
||||
use super::*;
|
||||
use crate::protocol::multimodal::{
|
||||
MmBatchedField, MmFeatureSpec, MmField, MmFieldElem, MmKwargValue, PlaceholderRange,
|
||||
};
|
||||
use crate::protocol::sampling::EngineCoreSamplingParams;
|
||||
use crate::protocol::tensor::{WireArrayData, WireTensor};
|
||||
use crate::protocol::{decode_value, encode_msgpack};
|
||||
|
||||
const AUX_FRAME_THRESHOLD: usize = 256;
|
||||
|
||||
#[test]
|
||||
fn engine_core_request_serializes_as_full_array() {
|
||||
let request = EngineCoreRequest {
|
||||
@@ -175,4 +192,84 @@ mod tests {
|
||||
assert_eq!(array[10], Value::Nil);
|
||||
assert_eq!(array[11], Value::from(7));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn engine_core_request_extracts_large_nested_tensors_in_wire_order() {
|
||||
let inline = vec![1_u8; AUX_FRAME_THRESHOLD - 1];
|
||||
let first_aux = vec![2_u8; AUX_FRAME_THRESHOLD];
|
||||
let second_aux = vec![3_u8; AUX_FRAME_THRESHOLD + 1];
|
||||
let first_aux_ptr = first_aux.as_ptr();
|
||||
let second_aux_ptr = second_aux.as_ptr();
|
||||
let mut request = EngineCoreRequest {
|
||||
mm_features: Some(vec![MmFeatureSpec {
|
||||
data: Some(BTreeMap::from([
|
||||
(
|
||||
"inline".to_string(),
|
||||
MmFieldElem {
|
||||
data: Some(MmKwargValue::Tensor(WireTensor::from_raw(
|
||||
"uint8",
|
||||
vec![inline.len()],
|
||||
inline,
|
||||
))),
|
||||
field: MmField::Batched(MmBatchedField { keep_on_cpu: false }),
|
||||
},
|
||||
),
|
||||
(
|
||||
"nested".to_string(),
|
||||
MmFieldElem {
|
||||
data: Some(MmKwargValue::List(vec![
|
||||
MmKwargValue::Int(7),
|
||||
MmKwargValue::Tensor(WireTensor::from_raw(
|
||||
"uint8",
|
||||
vec![first_aux.len()],
|
||||
first_aux,
|
||||
)),
|
||||
])),
|
||||
field: MmField::Batched(MmBatchedField { keep_on_cpu: false }),
|
||||
},
|
||||
),
|
||||
])),
|
||||
modality: "image".to_string(),
|
||||
identifier: "id".to_string(),
|
||||
mm_position: PlaceholderRange {
|
||||
offset: 0,
|
||||
length: second_aux.len(),
|
||||
is_embed: Some(WireTensor::from_raw(
|
||||
"bool",
|
||||
vec![second_aux.len()],
|
||||
second_aux,
|
||||
)),
|
||||
},
|
||||
mm_hash: None,
|
||||
}]),
|
||||
..EngineCoreRequest::default()
|
||||
};
|
||||
|
||||
let aux_frames = request.extract_aux_frames(AUX_FRAME_THRESHOLD);
|
||||
|
||||
assert_eq!(aux_frames.len(), 2);
|
||||
assert_eq!(aux_frames[0].as_ptr(), first_aux_ptr);
|
||||
assert_eq!(aux_frames[1].as_ptr(), second_aux_ptr);
|
||||
let feature = &request.mm_features.as_ref().unwrap()[0];
|
||||
let MmKwargValue::Tensor(inline) =
|
||||
feature.data.as_ref().unwrap()["inline"].data.as_ref().unwrap()
|
||||
else {
|
||||
panic!("expected inline tensor");
|
||||
};
|
||||
assert!(matches!(inline.data, WireArrayData::RawView(_)));
|
||||
let MmKwargValue::List(nested) =
|
||||
feature.data.as_ref().unwrap()["nested"].data.as_ref().unwrap()
|
||||
else {
|
||||
panic!("expected nested tensor list");
|
||||
};
|
||||
let MmKwargValue::Tensor(nested_tensor) = &nested[1] else {
|
||||
panic!("expected nested tensor");
|
||||
};
|
||||
assert_eq!(nested_tensor.data, WireArrayData::AuxIndex(1));
|
||||
assert_eq!(
|
||||
feature.mm_position.is_embed.as_ref().unwrap().data,
|
||||
WireArrayData::AuxIndex(2)
|
||||
);
|
||||
assert!(request.extract_aux_frames(AUX_FRAME_THRESHOLD).is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use bytemuck::allocation::pod_collect_to_vec;
|
||||
use bytemuck::{Pod, cast_slice};
|
||||
use bytes::Bytes;
|
||||
use enum_as_inner::EnumAsInner;
|
||||
use half::{bf16, f16};
|
||||
use rmpv::Value;
|
||||
@@ -20,6 +21,21 @@ struct MsgpackExtRef<'a>((i8, ByteSlice<'a>));
|
||||
|
||||
struct ByteSlice<'a>(&'a [u8]);
|
||||
|
||||
struct PodVec<T: Pod>(Vec<T>);
|
||||
|
||||
impl<T: Pod> AsRef<[u8]> for PodVec<T> {
|
||||
fn as_ref(&self) -> &[u8] {
|
||||
cast_slice(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
fn bytes_from_pod_vec<T>(data: Vec<T>) -> Bytes
|
||||
where
|
||||
T: Pod + Send + 'static,
|
||||
{
|
||||
Bytes::from_owner(PodVec(data))
|
||||
}
|
||||
|
||||
impl Serialize for ByteSlice<'_> {
|
||||
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
|
||||
where
|
||||
@@ -55,58 +71,63 @@ pub struct WireNdArray {
|
||||
|
||||
impl WireNdArray {
|
||||
/// Build a float32 tensor/ndarray backed by native-endian raw-view bytes.
|
||||
pub fn from_f32(shape: Vec<usize>, data: impl AsRef<[f32]>) -> Result<Self, String> {
|
||||
let data = data.as_ref();
|
||||
///
|
||||
/// Takes ownership of the backing buffer without copying its data.
|
||||
pub fn from_f32(shape: Vec<usize>, data: Vec<f32>) -> Result<Self, String> {
|
||||
validate_element_count(&shape, data.len())?;
|
||||
Ok(Self {
|
||||
dtype: "float32".to_string(),
|
||||
Ok(Self::from_raw_bytes(
|
||||
"float32",
|
||||
shape,
|
||||
data: WireArrayData::RawView(pod_collect_to_vec::<f32, u8>(data)),
|
||||
})
|
||||
bytes_from_pod_vec(data),
|
||||
))
|
||||
}
|
||||
|
||||
/// Build a float16 tensor/ndarray backed by native-endian raw-view bytes.
|
||||
pub fn from_f16(shape: Vec<usize>, data: impl AsRef<[f16]>) -> Result<Self, String> {
|
||||
let data = data.as_ref();
|
||||
///
|
||||
/// Takes ownership of the backing buffer without copying its data.
|
||||
pub fn from_f16(shape: Vec<usize>, data: Vec<f16>) -> Result<Self, String> {
|
||||
validate_element_count(&shape, data.len())?;
|
||||
Ok(Self {
|
||||
dtype: "float16".to_string(),
|
||||
Ok(Self::from_raw_bytes(
|
||||
"float16",
|
||||
shape,
|
||||
data: WireArrayData::RawView(pod_collect_to_vec::<f16, u8>(data)),
|
||||
})
|
||||
bytes_from_pod_vec(data),
|
||||
))
|
||||
}
|
||||
|
||||
/// Build a bfloat16 tensor/ndarray backed by native-endian raw-view bytes.
|
||||
pub fn from_bf16(shape: Vec<usize>, data: impl AsRef<[bf16]>) -> Result<Self, String> {
|
||||
let data = data.as_ref();
|
||||
///
|
||||
/// Takes ownership of the backing buffer without copying its data.
|
||||
pub fn from_bf16(shape: Vec<usize>, data: Vec<bf16>) -> Result<Self, String> {
|
||||
validate_element_count(&shape, data.len())?;
|
||||
Ok(Self {
|
||||
dtype: "bfloat16".to_string(),
|
||||
Ok(Self::from_raw_bytes(
|
||||
"bfloat16",
|
||||
shape,
|
||||
data: WireArrayData::RawView(pod_collect_to_vec::<bf16, u8>(data)),
|
||||
})
|
||||
bytes_from_pod_vec(data),
|
||||
))
|
||||
}
|
||||
|
||||
/// Build an int64 tensor/ndarray backed by native-endian raw-view bytes.
|
||||
pub fn from_i64(shape: Vec<usize>, data: impl AsRef<[i64]>) -> Result<Self, String> {
|
||||
let data = data.as_ref();
|
||||
///
|
||||
/// Takes ownership of the backing buffer without copying its data.
|
||||
pub fn from_i64(shape: Vec<usize>, data: Vec<i64>) -> Result<Self, String> {
|
||||
validate_element_count(&shape, data.len())?;
|
||||
Ok(Self {
|
||||
dtype: "int64".to_string(),
|
||||
Ok(Self::from_raw_bytes(
|
||||
"int64",
|
||||
shape,
|
||||
data: WireArrayData::RawView(pod_collect_to_vec::<i64, u8>(data)),
|
||||
})
|
||||
bytes_from_pod_vec(data),
|
||||
))
|
||||
}
|
||||
|
||||
/// Build a uint32 tensor/ndarray backed by native-endian raw-view bytes.
|
||||
pub fn from_u32(shape: Vec<usize>, data: impl AsRef<[u32]>) -> Result<Self, String> {
|
||||
let data = data.as_ref();
|
||||
///
|
||||
/// Takes ownership of the backing buffer without copying its data.
|
||||
pub fn from_u32(shape: Vec<usize>, data: Vec<u32>) -> Result<Self, String> {
|
||||
validate_element_count(&shape, data.len())?;
|
||||
Ok(Self {
|
||||
dtype: "uint32".to_string(),
|
||||
Ok(Self::from_raw_bytes(
|
||||
"uint32",
|
||||
shape,
|
||||
data: WireArrayData::RawView(pod_collect_to_vec::<u32, u8>(data)),
|
||||
})
|
||||
bytes_from_pod_vec(data),
|
||||
))
|
||||
}
|
||||
|
||||
/// Build a bool tensor/ndarray backed by raw-view bytes.
|
||||
@@ -118,7 +139,9 @@ impl WireNdArray {
|
||||
Ok(Self {
|
||||
dtype: "bool".to_string(),
|
||||
shape,
|
||||
data: WireArrayData::RawView(data.iter().map(|value| u8::from(*value)).collect()),
|
||||
data: WireArrayData::RawView(Bytes::from(
|
||||
data.into_iter().map(u8::from).collect::<Vec<_>>(),
|
||||
)),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -127,12 +150,22 @@ impl WireNdArray {
|
||||
/// Use this as an escape hatch when the caller already owns bytes that
|
||||
/// match the requested `dtype` and `shape`.
|
||||
pub fn from_raw(dtype: impl Into<String>, shape: Vec<usize>, data: Vec<u8>) -> Self {
|
||||
Self::from_raw_bytes(dtype, shape, Bytes::from(data))
|
||||
}
|
||||
|
||||
/// Build a tensor/ndarray from an owned immutable raw-view buffer.
|
||||
pub fn from_raw_bytes(dtype: impl Into<String>, shape: Vec<usize>, data: Bytes) -> Self {
|
||||
Self {
|
||||
dtype: dtype.into(),
|
||||
shape,
|
||||
data: WireArrayData::RawView(data),
|
||||
}
|
||||
}
|
||||
|
||||
/// Move a sufficiently large inline buffer into the ordered auxiliary-frame list.
|
||||
pub(crate) fn extract_aux_frame(&mut self, aux_frames: &mut Vec<Bytes>, threshold: usize) {
|
||||
self.data.extract_aux_frame(aux_frames, threshold);
|
||||
}
|
||||
}
|
||||
|
||||
/// Validate that the number of elements implied by the shape matches the length
|
||||
@@ -170,7 +203,25 @@ pub enum WireArrayData {
|
||||
/// stored.
|
||||
AuxIndex(usize),
|
||||
/// The raw bytes of this array/tensor.
|
||||
RawView(Vec<u8>),
|
||||
RawView(Bytes),
|
||||
}
|
||||
|
||||
impl WireArrayData {
|
||||
/// Replace a sufficiently large raw view with its one-based auxiliary-frame index.
|
||||
fn extract_aux_frame(&mut self, aux_frames: &mut Vec<Bytes>, threshold: usize) {
|
||||
let Self::RawView(bytes) = self else {
|
||||
return;
|
||||
};
|
||||
if bytes.len() < threshold {
|
||||
return;
|
||||
}
|
||||
|
||||
let index = aux_frames.len() + 1;
|
||||
let bytes = std::mem::replace(self, Self::AuxIndex(index))
|
||||
.into_raw_view()
|
||||
.expect("raw view was matched above");
|
||||
aux_frames.push(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for WireArrayData {
|
||||
@@ -180,7 +231,9 @@ impl<'de> Deserialize<'de> for WireArrayData {
|
||||
{
|
||||
let value = Value::deserialize(deserializer)?;
|
||||
match value {
|
||||
Value::Ext(tag, bytes) if tag == CUSTOM_TYPE_RAW_VIEW => Ok(Self::RawView(bytes)),
|
||||
Value::Ext(tag, bytes) if tag == CUSTOM_TYPE_RAW_VIEW => {
|
||||
Ok(Self::RawView(Bytes::from(bytes)))
|
||||
}
|
||||
Value::Ext(tag, _) => Err(serde::de::Error::custom(format!(
|
||||
"unsupported extension type code {tag}"
|
||||
))),
|
||||
@@ -201,9 +254,6 @@ impl Serialize for WireArrayData {
|
||||
where
|
||||
S: Serializer,
|
||||
{
|
||||
// TODO: outbound request serialization currently only supports inline
|
||||
// raw-view bytes. Emitting aux frames needs transport-level plumbing;
|
||||
// serializing `AuxIndex` here only preserves an already-built reference.
|
||||
match self {
|
||||
Self::AuxIndex(index) => serializer.serialize_u64(*index as u64),
|
||||
Self::RawView(bytes) => {
|
||||
@@ -221,7 +271,7 @@ mod tests {
|
||||
fn raw_view_serializes_as_msgpack_ext() {
|
||||
let bytes = vec![1, 2, 3, 4];
|
||||
let encoded =
|
||||
rmp_serde::to_vec_named(&WireArrayData::RawView(bytes.clone())).expect("encode");
|
||||
rmp_serde::to_vec_named(&WireArrayData::RawView(bytes.clone().into())).expect("encode");
|
||||
let expected = rmp_serde::to_vec_named(&Value::Ext(CUSTOM_TYPE_RAW_VIEW, bytes.clone()))
|
||||
.expect("encode expected");
|
||||
|
||||
@@ -234,11 +284,15 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn constructors_build_raw_view_tensors() {
|
||||
let f32_tensor = WireNdArray::from_f32(vec![2], vec![1.0, 2.5]).unwrap();
|
||||
let f32_data = vec![1.0, 2.5];
|
||||
let f32_data_ptr = f32_data.as_ptr().cast::<u8>();
|
||||
let f32_tensor = WireNdArray::from_f32(vec![2], f32_data).unwrap();
|
||||
assert_eq!(f32_tensor.dtype, "float32");
|
||||
assert_eq!(f32_tensor.shape, vec![2]);
|
||||
let f32_raw_view = f32_tensor.data.into_raw_view().expect("raw view");
|
||||
assert_eq!(f32_raw_view.as_ptr(), f32_data_ptr);
|
||||
assert_eq!(
|
||||
f32_tensor.data.into_raw_view().expect("raw view"),
|
||||
f32_raw_view,
|
||||
[1.0_f32, 2.5].into_iter().flat_map(f32::to_ne_bytes).collect::<Vec<_>>()
|
||||
);
|
||||
|
||||
@@ -258,15 +312,15 @@ mod tests {
|
||||
let i64_tensor = WireNdArray::from_i64(vec![1], vec![-7]).unwrap();
|
||||
assert_eq!(i64_tensor.dtype, "int64");
|
||||
assert_eq!(
|
||||
i64_tensor.data.into_raw_view().expect("raw view"),
|
||||
(-7_i64).to_ne_bytes()
|
||||
i64_tensor.data.into_raw_view().expect("raw view").as_ref(),
|
||||
(-7_i64).to_ne_bytes().as_ref()
|
||||
);
|
||||
|
||||
let u32_tensor = WireNdArray::from_u32(vec![1], vec![42]).unwrap();
|
||||
assert_eq!(u32_tensor.dtype, "uint32");
|
||||
assert_eq!(
|
||||
u32_tensor.data.into_raw_view().expect("raw view"),
|
||||
42_u32.to_ne_bytes()
|
||||
u32_tensor.data.into_raw_view().expect("raw view").as_ref(),
|
||||
42_u32.to_ne_bytes().as_ref()
|
||||
);
|
||||
|
||||
let bool_tensor = WireNdArray::from_bool(vec![2], vec![false, true]).unwrap();
|
||||
|
||||
@@ -32,7 +32,7 @@ use crate::protocol::output::{
|
||||
use crate::protocol::request::{EngineCoreRequest, EngineCoreRequestType};
|
||||
use crate::protocol::sampling::EngineCoreSamplingParams;
|
||||
use crate::protocol::stats::SchedulerStats;
|
||||
use crate::protocol::tensor::WireTensor;
|
||||
use crate::protocol::tensor::{WireArrayData, WireTensor};
|
||||
use crate::protocol::utility::{UtilityOutput, UtilityResultEnvelope};
|
||||
use crate::test_utils::{
|
||||
IpcNamespace, setup_bootstrapped_mock_engine, setup_mock_engine_sockets,
|
||||
@@ -1728,6 +1728,86 @@ async fn client_decodes_multipart_logprob_outputs() {
|
||||
client.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn client_sends_large_multimodal_tensor_as_aux_frame() {
|
||||
init_tracing();
|
||||
let ipc = IpcNamespace::new().unwrap();
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
let engine_id = b"engine-multimodal-aux".to_vec();
|
||||
let tensor_data = (0..64).map(|value| value as f32).collect::<Vec<_>>();
|
||||
let expected_bytes =
|
||||
tensor_data.iter().flat_map(|value| value.to_ne_bytes()).collect::<Vec<_>>();
|
||||
let mut request = sample_multimodal_request();
|
||||
request.mm_features.as_mut().unwrap()[0]
|
||||
.data
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.get_mut("pixel_values")
|
||||
.unwrap()
|
||||
.data = Some(MmKwargValue::Tensor(
|
||||
WireTensor::from_f32(vec![64], tensor_data).unwrap(),
|
||||
));
|
||||
|
||||
let (shutdown_tx, engine_task) = spawn_mock_engine_task(
|
||||
handshake_address.clone(),
|
||||
engine_id.clone(),
|
||||
move |dealer, push| {
|
||||
Box::pin(async move {
|
||||
let add = recv_engine_message(dealer).await;
|
||||
assert_eq!(add.len(), 3);
|
||||
assert_eq!(add[0].as_ref(), &[0x00]);
|
||||
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
|
||||
let MmKwargValue::Tensor(tensor) =
|
||||
request.mm_features.as_ref().unwrap()[0].data.as_ref().unwrap()["pixel_values"]
|
||||
.data
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
else {
|
||||
panic!("expected tensor");
|
||||
};
|
||||
assert_eq!(tensor.data, WireArrayData::AuxIndex(1));
|
||||
assert_eq!(add[2].as_ref(), expected_bytes);
|
||||
|
||||
send_outputs(
|
||||
push,
|
||||
RequestBatchOutputs {
|
||||
outputs: vec![request_output(
|
||||
"req-mm",
|
||||
vec![],
|
||||
Some(EngineCoreFinishReason::Length),
|
||||
)],
|
||||
finished_requests: Some(BTreeSet::from(["req-mm".to_string()])),
|
||||
..Default::default()
|
||||
}
|
||||
.into(),
|
||||
)
|
||||
.await;
|
||||
})
|
||||
},
|
||||
);
|
||||
|
||||
let client = connect_client_with_ipc(
|
||||
handshake_test_config(
|
||||
handshake_address,
|
||||
1,
|
||||
"test-model",
|
||||
Duration::from_secs(2),
|
||||
0,
|
||||
None,
|
||||
),
|
||||
&ipc,
|
||||
)
|
||||
.await;
|
||||
|
||||
let outputs = client.call(request).await.unwrap().collect::<Vec<_>>().await;
|
||||
assert_eq!(outputs.len(), 1);
|
||||
assert!(outputs[0].is_ok());
|
||||
|
||||
let _ = shutdown_tx.send(());
|
||||
engine_task.await.unwrap();
|
||||
client.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn multi_engine_client_shares_transport_and_routes_by_inflight_count() {
|
||||
init_tracing();
|
||||
|
||||
@@ -512,14 +512,14 @@ pub async fn send_message(
|
||||
input_send: &mut RouterSendHalf,
|
||||
engine_id: &EngineId,
|
||||
request_type: Bytes,
|
||||
payload: Vec<u8>,
|
||||
payload: Bytes,
|
||||
aux_frames: Vec<Bytes>,
|
||||
) -> Result<()> {
|
||||
let message = ZmqMessage::try_from(vec![
|
||||
engine_id.to_frame(),
|
||||
request_type,
|
||||
Bytes::from(payload),
|
||||
])
|
||||
.expect("router messages must contain identity and payload");
|
||||
let mut frames = Vec::with_capacity(3 + aux_frames.len());
|
||||
frames.extend([engine_id.to_frame(), request_type, payload]);
|
||||
frames.extend(aux_frames);
|
||||
let message =
|
||||
ZmqMessage::try_from(frames).expect("router messages must contain identity and payload");
|
||||
|
||||
trace!(
|
||||
?engine_id,
|
||||
|
||||
Reference in New Issue
Block a user