[Rust Frontend] Strongly type wire dtypes and multimodal modalities (#56174)

Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-09-10 02:11:22 +00:00
committed by GitHub
co-authored by Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 285cbce6bd
commit 41cffa9af0
15 changed files with 453 additions and 270 deletions
@@ -1,7 +1,191 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::fmt;
use std::str::FromStr;
use serde::{Deserialize, Serialize};
use serde_with::{DeserializeFromStr, SerializeDisplay};
/// Element types accepted by Python's tensor decoder.
#[derive(Debug, Clone, Copy, PartialEq, Eq, SerializeDisplay, DeserializeFromStr)]
pub enum TensorDtype {
Bool,
U8,
I8,
U16,
I16,
U32,
I32,
U64,
I64,
F16,
Bf16,
F32,
F64,
}
impl TensorDtype {
pub const fn as_str(self) -> &'static str {
match self {
Self::Bool => "bool",
Self::U8 => "uint8",
Self::I8 => "int8",
Self::U16 => "uint16",
Self::I16 => "int16",
Self::U32 => "uint32",
Self::I32 => "int32",
Self::U64 => "uint64",
Self::I64 => "int64",
Self::F16 => "float16",
Self::Bf16 => "bfloat16",
Self::F32 => "float32",
Self::F64 => "float64",
}
}
pub const fn element_size(self) -> usize {
match self {
Self::Bool | Self::U8 | Self::I8 => 1,
Self::U16 | Self::I16 | Self::F16 | Self::Bf16 => 2,
Self::U32 | Self::I32 | Self::F32 => 4,
Self::U64 | Self::I64 | Self::F64 => 8,
}
}
}
impl fmt::Display for TensorDtype {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.as_str())
}
}
impl FromStr for TensorDtype {
type Err = String;
fn from_str(value: &str) -> Result<Self, Self::Err> {
match value {
"bool" => Ok(Self::Bool),
"uint8" => Ok(Self::U8),
"int8" => Ok(Self::I8),
"uint16" => Ok(Self::U16),
"int16" => Ok(Self::I16),
"uint32" => Ok(Self::U32),
"int32" => Ok(Self::I32),
"uint64" => Ok(Self::U64),
"int64" => Ok(Self::I64),
"float16" => Ok(Self::F16),
"bfloat16" => Ok(Self::Bf16),
"float32" => Ok(Self::F32),
"float64" => Ok(Self::F64),
_ => Err(format!("unsupported tensor dtype {value:?}")),
}
}
}
/// Byte order carried by a NumPy dtype descriptor.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Endianness {
Little,
Big,
Native,
}
/// Numeric array dtype with an explicit byte-order contract.
///
/// Also accepts Torch type names as native-endian dtypes because prompt
/// logprobs use tensor tuples with the same wire shape as NumPy arrays.
/// Serializes byte order explicitly, including `=` for native byte order.
/// Bfloat16 uses the Torch name after the prefix as a nonstandard descriptor.
#[derive(Debug, Clone, Copy, PartialEq, Eq, SerializeDisplay, DeserializeFromStr)]
pub struct NumpyDtype {
pub scalar: TensorDtype,
pub endianness: Endianness,
}
impl NumpyDtype {
pub const fn little(scalar: TensorDtype) -> Self {
Self {
scalar,
endianness: Endianness::Little,
}
}
pub const fn native(scalar: TensorDtype) -> Self {
Self {
scalar,
endianness: Endianness::Native,
}
}
pub const fn big(scalar: TensorDtype) -> Self {
Self {
scalar,
endianness: Endianness::Big,
}
}
}
impl From<TensorDtype> for NumpyDtype {
fn from(scalar: TensorDtype) -> Self {
Self::native(scalar)
}
}
impl fmt::Display for NumpyDtype {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let prefix = match self.endianness {
Endianness::Native => '=',
Endianness::Little => '<',
Endianness::Big => '>',
};
let code = match self.scalar {
TensorDtype::Bool => "b1",
TensorDtype::U8 => "u1",
TensorDtype::I8 => "i1",
TensorDtype::U16 => "u2",
TensorDtype::I16 => "i2",
TensorDtype::U32 => "u4",
TensorDtype::I32 => "i4",
TensorDtype::U64 => "u8",
TensorDtype::I64 => "i8",
TensorDtype::F16 => "f2",
TensorDtype::F32 => "f4",
TensorDtype::F64 => "f8",
TensorDtype::Bf16 => "bfloat16",
};
write!(f, "{prefix}{code}")
}
}
impl FromStr for NumpyDtype {
type Err = String;
fn from_str(dtype: &str) -> Result<Self, Self::Err> {
let (endianness, body) = match dtype.as_bytes().first().copied() {
Some(b'<') => (Endianness::Little, &dtype[1..]),
Some(b'>') => (Endianness::Big, &dtype[1..]),
Some(b'=' | b'|') => (Endianness::Native, &dtype[1..]),
_ => (Endianness::Native, dtype),
};
let scalar = match body {
"b1" => TensorDtype::Bool,
"u1" => TensorDtype::U8,
"i1" => TensorDtype::I8,
"u2" => TensorDtype::U16,
"i2" => TensorDtype::I16,
"u4" => TensorDtype::U32,
"i4" => TensorDtype::I32,
"u8" => TensorDtype::U64,
"i8" => TensorDtype::I64,
"f2" => TensorDtype::F16,
"f4" => TensorDtype::F32,
"f8" => TensorDtype::F64,
_ => body.parse().map_err(|_| format!("unsupported dtype string {dtype:?}"))?,
};
Ok(Self { scalar, endianness })
}
}
/// Effective model dtype reported by the engine after config resolution.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
@@ -26,7 +210,53 @@ impl ModelDtype {
#[cfg(test)]
mod tests {
use super::ModelDtype;
use super::{Endianness, ModelDtype, NumpyDtype, TensorDtype};
#[test]
fn numpy_descriptors_preserve_explicit_byte_order() {
for (code, scalar) in [
("b1", TensorDtype::Bool),
("u1", TensorDtype::U8),
("i1", TensorDtype::I8),
("u2", TensorDtype::U16),
("i2", TensorDtype::I16),
("u4", TensorDtype::U32),
("i4", TensorDtype::I32),
("u8", TensorDtype::U64),
("i8", TensorDtype::I64),
("f2", TensorDtype::F16),
("bfloat16", TensorDtype::Bf16),
("f4", TensorDtype::F32),
("f8", TensorDtype::F64),
] {
for (prefix, endianness) in [("<", Endianness::Little), (">", Endianness::Big)] {
let wire = serde_json::json!(format!("{prefix}{code}"));
let dtype: NumpyDtype = serde_json::from_value(wire.clone()).unwrap();
assert_eq!(dtype, NumpyDtype { scalar, endianness });
assert_eq!(serde_json::to_value(dtype).unwrap(), wire);
}
for wire in [
format!("={code}"),
format!("|{code}"),
code.into(),
scalar.as_str().into(),
] {
let dtype: NumpyDtype = serde_json::from_value(serde_json::json!(wire)).unwrap();
assert_eq!(dtype, NumpyDtype::from(scalar));
assert_eq!(
serde_json::to_value(dtype).unwrap(),
serde_json::json!(format!("={code}"))
);
}
}
}
#[test]
fn rejects_unsupported_numpy_dtypes() {
for wire in ["", "object", "<c8", "<bf2"] {
assert!(serde_json::from_value::<NumpyDtype>(serde_json::json!(wire)).is_err());
}
}
#[test]
fn serde_uses_protocol_dtype_strings() {
@@ -14,6 +14,7 @@ use serde::{Deserialize, Deserializer, Serialize};
use self::wire::*;
use crate::error::{Error, Result, bail_ext_value_decode};
use crate::protocol::dtype::{NumpyDtype, TensorDtype};
use crate::protocol::tensor::{WireArrayData, WireNdArray};
/// One token candidate and its logprob metadata for a single sequence position.
@@ -205,17 +206,17 @@ impl WireLogprobs {
Ok(Self {
logprob_token_ids: WireNdArray {
dtype: "<i8".to_string(),
dtype: NumpyDtype::little(TensorDtype::I64),
shape: vec![rows, cols],
data: WireArrayData::RawView(token_ids.into()),
},
logprobs: WireNdArray {
dtype: "<f4".to_string(),
dtype: NumpyDtype::little(TensorDtype::F32),
shape: vec![rows, cols],
data: WireArrayData::RawView(logprobs.into()),
},
token_ranks: WireNdArray {
dtype: "<i8".to_string(),
dtype: NumpyDtype::little(TensorDtype::I64),
shape: vec![rows],
data: WireArrayData::RawView(token_ranks.into()),
},
@@ -8,22 +8,9 @@ use bytes::Bytes;
use itertools::Itertools as _;
use crate::error::{Error, Result, ext_value_decode};
use crate::protocol::dtype::{Endianness, TensorDtype};
use crate::protocol::tensor::{ShapeExt as _, WireNdArray};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) enum ScalarType {
I32,
I64,
F32,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) enum Endianness {
Little,
Big,
Native,
}
#[derive(Debug, Clone, PartialEq)]
pub(super) struct DecodedArray2<T> {
pub rows: usize,
@@ -37,7 +24,7 @@ pub(super) fn decode_array2_u32(
frames: &[Bytes],
) -> Result<DecodedArray2<u32>> {
let (shape, bytes, scalar, endianness) =
decode_array_metadata(value, field, frames, &[ScalarType::I32, ScalarType::I64])?;
decode_array_metadata(value, field, frames, &[TensorDtype::I32, TensorDtype::I64])?;
if shape.len() != 2 {
return Err(decode_error(
field,
@@ -46,15 +33,15 @@ pub(super) fn decode_array2_u32(
}
let data = match scalar {
ScalarType::I32 => decode_i32_vec(&bytes, endianness, field)?
TensorDtype::I32 => decode_i32_vec(&bytes, endianness, field)?
.into_iter()
.map(|value| convert_to_u32(value, field))
.try_collect()?,
ScalarType::I64 => decode_i64_vec(&bytes, endianness, field)?
TensorDtype::I64 => decode_i64_vec(&bytes, endianness, field)?
.into_iter()
.map(|value| convert_to_u32(value, field))
.try_collect()?,
ScalarType::F32 => unreachable!("scalar validation should reject f32"),
_ => unreachable!("scalar validation should accept only i32 and i64"),
};
Ok(DecodedArray2 {
rows: shape[0],
@@ -69,7 +56,7 @@ pub(super) fn decode_array1_u32(
frames: &[Bytes],
) -> Result<Vec<u32>> {
let (shape, bytes, scalar, endianness) =
decode_array_metadata(value, field, frames, &[ScalarType::I32, ScalarType::I64])?;
decode_array_metadata(value, field, frames, &[TensorDtype::I32, TensorDtype::I64])?;
if shape.len() != 1 {
return Err(decode_error(
field,
@@ -78,15 +65,15 @@ pub(super) fn decode_array1_u32(
}
let data = match scalar {
ScalarType::I32 => decode_i32_vec(&bytes, endianness, field)?
TensorDtype::I32 => decode_i32_vec(&bytes, endianness, field)?
.into_iter()
.map(|value| convert_to_u32(value, field))
.try_collect()?,
ScalarType::I64 => decode_i64_vec(&bytes, endianness, field)?
TensorDtype::I64 => decode_i64_vec(&bytes, endianness, field)?
.into_iter()
.map(|value| convert_to_u32(value, field))
.try_collect()?,
ScalarType::F32 => unreachable!("scalar validation should reject f32"),
_ => unreachable!("scalar validation should accept only i32 and i64"),
};
Ok(data)
}
@@ -97,7 +84,7 @@ pub(super) fn decode_array2_f32(
frames: &[Bytes],
) -> Result<DecodedArray2<f32>> {
let (shape, bytes, _, endianness) =
decode_array_metadata(value, field, frames, &[ScalarType::F32])?;
decode_array_metadata(value, field, frames, &[TensorDtype::F32])?;
if shape.len() != 2 {
return Err(decode_error(
field,
@@ -117,15 +104,16 @@ pub(super) fn decode_array_metadata(
value: WireNdArray,
field: &str,
frames: &[Bytes],
expected_scalars: &[ScalarType],
) -> Result<(Vec<usize>, Bytes, ScalarType, Endianness)> {
expected_scalars: &[TensorDtype],
) -> Result<(Vec<usize>, Bytes, TensorDtype, Endianness)> {
let mut value = value;
let (scalar, endianness) = parse_dtype(&value.dtype, field)?;
let scalar = value.dtype.scalar;
let endianness = value.dtype.endianness;
if !expected_scalars.contains(&scalar) {
return Err(decode_error(
field,
&format!(
"expected dtype in {:?}, got {}",
"expected dtype in {:?}, got {:?}",
expected_scalars, value.dtype
),
));
@@ -140,42 +128,16 @@ pub(super) fn decode_array_metadata(
Ok((shape, bytes, scalar, endianness))
}
pub(super) fn parse_dtype(dtype: &str, field: &str) -> Result<(ScalarType, Endianness)> {
let (endianness, body) = match dtype.as_bytes().first().copied() {
Some(b'<') => (Endianness::Little, &dtype[1..]),
Some(b'>') => (Endianness::Big, &dtype[1..]),
Some(b'=') => (Endianness::Native, &dtype[1..]),
Some(b'|') => (Endianness::Native, &dtype[1..]),
_ => (Endianness::Native, dtype),
};
let scalar = match body {
"i4" | "int32" => ScalarType::I32,
"i8" | "int64" => ScalarType::I64,
"f4" | "float32" => ScalarType::F32,
_ => {
return Err(decode_error(
field,
&format!("unsupported dtype string {dtype:?}"),
));
}
};
Ok((scalar, endianness))
}
pub(super) fn validate_byte_length(
shape: &[usize],
byte_len: usize,
field: &str,
scalar: ScalarType,
scalar: TensorDtype,
) -> Result<()> {
let element_count = shape
.checked_numel()
.ok_or_else(|| decode_error(field, "shape element count overflowed usize"))?;
let element_size = match scalar {
ScalarType::I32 | ScalarType::F32 => 4,
ScalarType::I64 => 8,
};
let element_size = scalar.element_size();
let expected = element_count
.checked_mul(element_size)
.ok_or_else(|| decode_error(field, "byte length overflowed usize"))?;
@@ -302,6 +302,29 @@ fn decodes_big_endian_payloads() {
);
}
#[test]
fn rejects_supported_array_dtypes_in_incompatible_logprobs_fields() {
for (ids_dtype, probs_dtype, field) in [
("<u4", "<f4", "logprob_token_ids"),
("<i4", "<i4", "logprobs"),
] {
let frames = vec![Bytes::from(encode_value(&output_wire_with_custom_fields(
Some(Value::Array(vec![
ndarray_value(ids_dtype, &[1, 1], Value::Ext(3, vec![0; 4])),
ndarray_value(probs_dtype, &[1, 1], Value::Ext(3, vec![0; 4])),
ndarray_value("<i4", &[1], Value::Ext(3, vec![1, 0, 0, 0])),
Value::Nil,
])),
None,
)))];
let error = decode_engine_core_outputs(&frames).unwrap_err();
let crate::error::Error::ExtValueDecode { message } = error else {
panic!("expected ExtValueDecode");
};
assert!(message.starts_with(&format!("new_logprobs.{field}: expected dtype")));
}
}
#[test]
fn rejects_non_none_cu_num_generated_tokens() {
let frames = vec![Bytes::from(encode_value(&output_wire_with_custom_fields(
@@ -9,6 +9,26 @@ use serde_tuple::{Deserialize_tuple, Serialize_tuple};
use super::tensor::WireTensor;
/// Modalities represented by engine-ready multimodal features.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum MmModality {
Image,
Audio,
Video,
}
impl MmModality {
/// The wire name, matching Python's modality strings.
pub fn as_str(self) -> &'static str {
match self {
Self::Image => "image",
Self::Audio => "audio",
Self::Video => "video",
}
}
}
/// Multimodal feature payload accepted from higher-level frontend code.
///
/// Original Python definition:
@@ -32,7 +52,7 @@ pub struct MmFeatureSpec {
pub data: Option<MmKwargsItem>,
/// The input modality, e.g., `"image"`, `"audio"`, `"video"`.
pub modality: String,
pub modality: MmModality,
/// The hash for caching encoder outputs (with LoRA prefix if applicable).
pub identifier: String,
@@ -158,8 +158,10 @@ mod tests {
use rmpv::Value;
use super::*;
use crate::protocol::dtype::TensorDtype;
use crate::protocol::multimodal::{
MmBatchedField, MmFeatureSpec, MmField, MmFieldElem, MmKwargValue, PlaceholderRange,
MmBatchedField, MmFeatureSpec, MmField, MmFieldElem, MmKwargValue, MmModality,
PlaceholderRange,
};
use crate::protocol::sampling::EngineCoreSamplingParams;
use crate::protocol::tensor::{WireArrayData, WireTensor};
@@ -212,7 +214,7 @@ mod tests {
"inline".to_string(),
MmFieldElem {
data: Some(MmKwargValue::Tensor(WireTensor::from_raw(
"uint8",
TensorDtype::U8,
vec![inline.len()],
inline,
))),
@@ -225,7 +227,7 @@ mod tests {
data: Some(MmKwargValue::List(vec![
MmKwargValue::Int(7),
MmKwargValue::Tensor(WireTensor::from_raw(
"uint8",
TensorDtype::U8,
vec![first_aux.len()],
first_aux,
)),
@@ -234,13 +236,13 @@ mod tests {
},
),
])),
modality: "image".to_string(),
modality: MmModality::Image,
identifier: "id".to_string(),
mm_position: PlaceholderRange {
offset: 0,
length: second_aux.len(),
is_embed: Some(WireTensor::from_raw(
"bool",
TensorDtype::Bool,
vec![second_aux.len()],
second_aux,
)),
@@ -6,9 +6,12 @@ use bytes::Bytes;
use enum_as_inner::EnumAsInner;
use half::{bf16, f16};
use rmpv::Value;
use serde::de::DeserializeOwned;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
use super::dtype::{NumpyDtype, TensorDtype};
/// Tensors and ndarrays are encoded with this extension type in Python.
///
/// Original Python definition:
@@ -63,20 +66,20 @@ impl [usize] {
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/v1/serial_utils.py#L237-L273>
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/v1/serial_utils.py#L389-L425>
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple)]
pub struct WireNdArray {
pub dtype: String,
pub struct WireArray<Dtype: Serialize + DeserializeOwned> {
pub dtype: Dtype,
pub shape: Vec<usize>,
pub data: WireArrayData,
}
impl WireNdArray {
impl<Dtype: From<TensorDtype> + Serialize + DeserializeOwned> WireArray<Dtype> {
/// Build a float32 tensor/ndarray backed by native-endian raw-view bytes.
///
/// Takes ownership of the backing buffer without copying its data.
pub fn from_f32(shape: Vec<usize>, data: Vec<f32>) -> Result<Self, String> {
validate_element_count(&shape, data.len())?;
Ok(Self::from_raw_bytes(
"float32",
TensorDtype::F32,
shape,
bytes_from_pod_vec(data),
))
@@ -88,7 +91,7 @@ impl WireNdArray {
pub fn from_f16(shape: Vec<usize>, data: Vec<f16>) -> Result<Self, String> {
validate_element_count(&shape, data.len())?;
Ok(Self::from_raw_bytes(
"float16",
TensorDtype::F16,
shape,
bytes_from_pod_vec(data),
))
@@ -100,7 +103,7 @@ impl WireNdArray {
pub fn from_bf16(shape: Vec<usize>, data: Vec<bf16>) -> Result<Self, String> {
validate_element_count(&shape, data.len())?;
Ok(Self::from_raw_bytes(
"bfloat16",
TensorDtype::Bf16,
shape,
bytes_from_pod_vec(data),
))
@@ -112,7 +115,7 @@ impl WireNdArray {
pub fn from_i64(shape: Vec<usize>, data: Vec<i64>) -> Result<Self, String> {
validate_element_count(&shape, data.len())?;
Ok(Self::from_raw_bytes(
"int64",
TensorDtype::I64,
shape,
bytes_from_pod_vec(data),
))
@@ -124,7 +127,7 @@ impl WireNdArray {
pub fn from_u32(shape: Vec<usize>, data: Vec<u32>) -> Result<Self, String> {
validate_element_count(&shape, data.len())?;
Ok(Self::from_raw_bytes(
"uint32",
TensorDtype::U32,
shape,
bytes_from_pod_vec(data),
))
@@ -137,7 +140,7 @@ impl WireNdArray {
pub fn from_bool(shape: Vec<usize>, data: Vec<bool>) -> Result<Self, String> {
validate_element_count(&shape, data.len())?;
Ok(Self {
dtype: "bool".to_string(),
dtype: TensorDtype::Bool.into(),
shape,
data: WireArrayData::RawView(Bytes::from(
data.into_iter().map(u8::from).collect::<Vec<_>>(),
@@ -149,12 +152,12 @@ impl WireNdArray {
///
/// Use this as an escape hatch when the caller already owns bytes that
/// match the requested `dtype` and `shape`.
pub fn from_raw(dtype: impl Into<String>, shape: Vec<usize>, data: Vec<u8>) -> Self {
pub fn from_raw(dtype: impl Into<Dtype>, shape: Vec<usize>, data: Vec<u8>) -> Self {
Self::from_raw_bytes(dtype, shape, Bytes::from(data))
}
/// Build a tensor/ndarray from an owned immutable raw-view buffer.
pub fn from_raw_bytes(dtype: impl Into<String>, shape: Vec<usize>, data: Bytes) -> Self {
pub fn from_raw_bytes(dtype: impl Into<Dtype>, shape: Vec<usize>, data: Bytes) -> Self {
Self {
dtype: dtype.into(),
shape,
@@ -192,7 +195,10 @@ fn validate_element_count(shape: &[usize], len: usize) -> Result<(), String> {
///
/// This is the same wire shape as [`WireNdArray`]; multimodal request payloads
/// use it for `torch.Tensor` values.
pub type WireTensor = WireNdArray;
pub type WireTensor = WireArray<TensorDtype>;
/// Numeric array with a typed NumPy descriptor or native-endian Torch dtype.
pub type WireNdArray = WireArray<NumpyDtype>;
/// Python array/tensor payload reference inside [`WireNdArray`].
///
@@ -211,6 +217,27 @@ pub enum WireArrayData {
RawView(Bytes),
}
impl WireTensor {
/// Validate the shape and storage of an inline tensor.
pub fn validate_inline(&self) -> Result<(), String> {
let expected = self
.shape
.checked_numel()
.and_then(|numel| numel.checked_mul(self.dtype.element_size()))
.ok_or_else(|| format!("tensor shape or byte length overflows: {:?}", self.shape))?;
match &self.data {
WireArrayData::RawView(bytes) if bytes.len() == expected => Ok(()),
WireArrayData::RawView(bytes) => Err(format!(
"tensor byte length {} does not match expected {expected}",
bytes.len()
)),
WireArrayData::AuxIndex(index) => {
Err(format!("inline tensor references auxiliary frame {index}"))
}
}
}
}
impl WireArrayData {
/// Resolve an auxiliary-frame reference into owned raw bytes.
pub(crate) fn resolve_aux_frame(&mut self, frames: &[Bytes]) -> Result<(), String> {
@@ -288,6 +315,21 @@ impl Serialize for WireArrayData {
mod tests {
use super::*;
#[test]
fn tensor_and_ndarray_preserve_their_dtype_wire_contracts() {
let tensor = WireTensor::from_f32(vec![1], vec![1.0]).unwrap();
let bytes = rmp_serde::to_vec_named(&tensor).unwrap();
let wire: Value = rmp_serde::from_slice(&bytes).unwrap();
assert_eq!(wire.as_array().unwrap()[0], Value::from("float32"));
let array: WireNdArray = rmp_serde::from_slice(&bytes).unwrap();
assert_eq!(array.dtype, NumpyDtype::native(TensorDtype::F32));
let bytes = rmp_serde::to_vec_named(&array).unwrap();
let wire: Value = rmp_serde::from_slice(&bytes).unwrap();
assert_eq!(wire.as_array().unwrap()[0], Value::from("=f4"));
assert!(rmp_serde::from_slice::<WireTensor>(&bytes).is_err());
}
#[test]
fn raw_view_serializes_as_msgpack_ext() {
let bytes = vec![1, 2, 3, 4];
@@ -307,8 +349,8 @@ mod tests {
fn constructors_build_raw_view_tensors() {
let f32_data = vec![1.0, 2.5];
let f32_data_ptr = f32_data.as_ptr().cast::<u8>();
let f32_tensor = WireNdArray::from_f32(vec![2], f32_data).unwrap();
assert_eq!(f32_tensor.dtype, "float32");
let f32_tensor = WireTensor::from_f32(vec![2], f32_data).unwrap();
assert_eq!(f32_tensor.dtype, TensorDtype::F32);
assert_eq!(f32_tensor.shape, vec![2]);
let f32_raw_view = f32_tensor.data.into_raw_view().expect("raw view");
assert_eq!(f32_raw_view.as_ptr(), f32_data_ptr);
@@ -318,58 +360,49 @@ mod tests {
);
let f16_tensor =
WireNdArray::from_f16(vec![2], vec![f16::from_f32(1.0), f16::from_f32(2.5)]).unwrap();
assert_eq!(f16_tensor.dtype, "float16");
WireTensor::from_f16(vec![2], vec![f16::from_f32(1.0), f16::from_f32(2.5)]).unwrap();
assert_eq!(f16_tensor.dtype, TensorDtype::F16);
assert_eq!(f16_tensor.shape, vec![2]);
assert_eq!(f16_tensor.data.into_raw_view().expect("raw view").len(), 4);
let bf16_tensor =
WireNdArray::from_bf16(vec![2], vec![bf16::from_f32(1.0), bf16::from_f32(2.5)])
.unwrap();
assert_eq!(bf16_tensor.dtype, "bfloat16");
WireTensor::from_bf16(vec![2], vec![bf16::from_f32(1.0), bf16::from_f32(2.5)]).unwrap();
assert_eq!(bf16_tensor.dtype, TensorDtype::Bf16);
assert_eq!(bf16_tensor.shape, vec![2]);
assert_eq!(bf16_tensor.data.into_raw_view().expect("raw view").len(), 4);
let i64_tensor = WireNdArray::from_i64(vec![1], vec![-7]).unwrap();
assert_eq!(i64_tensor.dtype, "int64");
let i64_tensor = WireTensor::from_i64(vec![1], vec![-7]).unwrap();
assert_eq!(i64_tensor.dtype, TensorDtype::I64);
assert_eq!(
i64_tensor.data.into_raw_view().expect("raw view").as_ref(),
(-7_i64).to_ne_bytes().as_ref()
);
let u32_tensor = WireNdArray::from_u32(vec![1], vec![42]).unwrap();
assert_eq!(u32_tensor.dtype, "uint32");
let u32_tensor = WireTensor::from_u32(vec![1], vec![42]).unwrap();
assert_eq!(u32_tensor.dtype, TensorDtype::U32);
assert_eq!(
u32_tensor.data.into_raw_view().expect("raw view").as_ref(),
42_u32.to_ne_bytes().as_ref()
);
let bool_tensor = WireNdArray::from_bool(vec![2], vec![false, true]).unwrap();
assert_eq!(bool_tensor.dtype, "bool");
let bool_tensor = WireTensor::from_bool(vec![2], vec![false, true]).unwrap();
assert_eq!(bool_tensor.dtype, TensorDtype::Bool);
assert_eq!(
bool_tensor.data.into_raw_view().expect("raw view"),
vec![0, 1]
);
let raw_tensor = WireNdArray::from_raw("custom", vec![3], vec![1, 2, 3]);
assert_eq!(raw_tensor.dtype, "custom");
assert_eq!(raw_tensor.shape, vec![3]);
assert_eq!(
raw_tensor.data.into_raw_view().expect("raw view"),
vec![1, 2, 3]
);
}
#[test]
fn constructors_validate_shape_product() {
let err = WireNdArray::from_f32(vec![2, 2], vec![1.0, 2.0]).unwrap_err();
let err = WireTensor::from_f32(vec![2, 2], vec![1.0, 2.0]).unwrap_err();
assert!(err.contains("does not match shape"));
}
#[test]
fn resolve_aux_frame_shares_bytes_using_one_based_message_index() {
let mut tensor = WireNdArray {
dtype: "float32".to_string(),
let mut tensor = WireTensor {
dtype: TensorDtype::F32,
shape: vec![2],
data: WireArrayData::AuxIndex(1),
};
@@ -22,8 +22,8 @@ use zeromq::{DealerSocket, PushSocket, SocketOptions, SubSocket, XPubSocket, Zmq
use crate::protocol::handshake::{EngineCoreReadyResponse, HandshakeInitMessage, ReadyMessage};
use crate::protocol::logprobs::MaybeWireLogprobs;
use crate::protocol::multimodal::{
MmFeatureSpec, MmField, MmFieldElem, MmFlatField, MmKwargValue, MmSlice, PlaceholderRange,
SliceSpec,
MmFeatureSpec, MmField, MmFieldElem, MmFlatField, MmKwargValue, MmModality, MmSlice,
PlaceholderRange, SliceSpec,
};
use crate::protocol::output::{
DpControlMessage, DpControlOutput, EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs,
@@ -191,7 +191,7 @@ fn sample_multimodal_request() -> EngineCoreRequest {
}),
},
)])),
modality: "image".to_string(),
modality: MmModality::Image,
identifier: "mm-cache-key".to_string(),
mm_position: PlaceholderRange {
offset: 1,