[Rust Frontend] Strongly type wire dtypes and multimodal modalities (#56174)
Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
285cbce6bd
commit
41cffa9af0
@@ -60,30 +60,9 @@ pub struct MultimodalModelInfo {
|
||||
/// Per-modality item-count limits configured by `--limit-mm-per-prompt`.
|
||||
///
|
||||
/// Modalities absent from the map are unlimited.
|
||||
pub type MmLimitPerPrompt = HashMap<MmLimitModality, MmLimitSpec>;
|
||||
pub type MmLimitPerPrompt = HashMap<MmModality, MmLimitSpec>;
|
||||
|
||||
/// Modalities that `--limit-mm-per-prompt` can be keyed by.
|
||||
///
|
||||
/// Closed on purpose: these are exactly the keys Python accepts, per
|
||||
/// `MultiModalDummyOptionsBuiltins` in `vllm/config/multimodal.py`.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum MmLimitModality {
|
||||
Image,
|
||||
Audio,
|
||||
Video,
|
||||
}
|
||||
|
||||
impl MmLimitModality {
|
||||
/// The wire name, matching Python's modality strings.
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Image => "image",
|
||||
Self::Audio => "audio",
|
||||
Self::Video => "video",
|
||||
}
|
||||
}
|
||||
}
|
||||
pub use vllm_engine_core_client::protocol::multimodal::MmModality;
|
||||
|
||||
/// One modality's limit, in either of the two shapes Python accepts.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
@@ -646,17 +625,17 @@ fn input_audio_data_url(data: &str, format: Option<&str>) -> Result<String> {
|
||||
/// Embedding inputs share their base modality's budget rather than getting one
|
||||
/// of their own, matching Python's `modality.replace("_embeds", "")` in
|
||||
/// `vllm/entrypoints/chat_utils.py`.
|
||||
fn media_part_limit_modality(part: &MediaContentPart) -> Option<MmLimitModality> {
|
||||
fn media_part_limit_modality(part: &MediaContentPart) -> Option<MmModality> {
|
||||
match part {
|
||||
MediaContentPart::Text { .. } => None,
|
||||
MediaContentPart::ImageUrl { .. }
|
||||
| MediaContentPart::ImageData { .. }
|
||||
| MediaContentPart::ImageEmbeds { .. } => Some(MmLimitModality::Image),
|
||||
| MediaContentPart::ImageEmbeds { .. } => Some(MmModality::Image),
|
||||
MediaContentPart::AudioUrl { .. } | MediaContentPart::AudioData { .. } => {
|
||||
Some(MmLimitModality::Audio)
|
||||
Some(MmModality::Audio)
|
||||
}
|
||||
MediaContentPart::VideoUrl { .. } | MediaContentPart::VideoData { .. } => {
|
||||
Some(MmLimitModality::Video)
|
||||
Some(MmModality::Video)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -667,7 +646,7 @@ impl MultimodalModelInfo {
|
||||
///
|
||||
/// Modalities without a configured count are unlimited.
|
||||
fn validate_mm_limits(&self, media_parts: &[MediaContentPart]) -> Result<()> {
|
||||
let mut counts: HashMap<MmLimitModality, usize> = HashMap::new();
|
||||
let mut counts: HashMap<MmModality, usize> = HashMap::new();
|
||||
for part in media_parts {
|
||||
if let Some(modality) = media_part_limit_modality(part) {
|
||||
*counts.entry(modality).or_default() += 1;
|
||||
@@ -738,7 +717,11 @@ impl MultimodalModelInfo {
|
||||
for (item, range) in izip!(media.items, media_ranges) {
|
||||
features.push(MmFeatureSpec {
|
||||
data: Some(item.data),
|
||||
modality: media.modality.to_string(),
|
||||
modality: match media.modality {
|
||||
Modality::Image | Modality::ImageEmbeds => MmModality::Image,
|
||||
Modality::Video => MmModality::Video,
|
||||
Modality::Audio => MmModality::Audio,
|
||||
},
|
||||
identifier: item.uuid.unwrap_or_else(|| item.hash.clone()),
|
||||
mm_position: range,
|
||||
mm_hash: Some(item.hash),
|
||||
@@ -1029,10 +1012,8 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn validate_mm_limits_enforces_configured_limit_at_the_boundary() {
|
||||
let info = qwen3_vl_info_with_limits(HashMap::from([(
|
||||
MmLimitModality::Image,
|
||||
MmLimitSpec::Count(1),
|
||||
)]));
|
||||
let info =
|
||||
qwen3_vl_info_with_limits(HashMap::from([(MmModality::Image, MmLimitSpec::Count(1))]));
|
||||
assert!(info.validate_mm_limits(&[image_url_part()]).is_ok());
|
||||
|
||||
let error = info.validate_mm_limits(&[image_url_part(), image_url_part()]).unwrap_err();
|
||||
@@ -1047,10 +1028,8 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn validate_mm_limits_counts_image_embeds_against_the_image_limit() {
|
||||
let info = qwen3_vl_info_with_limits(HashMap::from([(
|
||||
MmLimitModality::Image,
|
||||
MmLimitSpec::Count(1),
|
||||
)]));
|
||||
let info =
|
||||
qwen3_vl_info_with_limits(HashMap::from([(MmModality::Image, MmLimitSpec::Count(1))]));
|
||||
let image_embeds_part = MediaContentPart::ImageEmbeds {
|
||||
payload: serde_json::Value::String("AAAA".to_string()),
|
||||
uuid: None,
|
||||
@@ -1068,7 +1047,7 @@ mod tests {
|
||||
#[test]
|
||||
fn validate_mm_limits_treats_a_count_less_options_object_as_unlimited() {
|
||||
let info = qwen3_vl_info_with_limits(HashMap::from([(
|
||||
MmLimitModality::Image,
|
||||
MmModality::Image,
|
||||
MmLimitSpec::Options {
|
||||
count: None,
|
||||
extra: BTreeMap::from([("width".to_string(), serde_json::json!(512))]),
|
||||
@@ -1086,9 +1065,9 @@ mod tests {
|
||||
fn limit_map_parses_both_shapes_python_accepts() {
|
||||
let limits = parse_limits(r#"{"image": 16, "video": {"count": 1, "num_frames": 32}}"#);
|
||||
|
||||
assert_eq!(limits[&MmLimitModality::Image].count(), Some(16));
|
||||
assert_eq!(limits[&MmLimitModality::Video].count(), Some(1));
|
||||
assert_eq!(limits.get(&MmLimitModality::Audio), None);
|
||||
assert_eq!(limits[&MmModality::Image].count(), Some(16));
|
||||
assert_eq!(limits[&MmModality::Video].count(), Some(1));
|
||||
assert_eq!(limits.get(&MmModality::Audio), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -268,14 +268,14 @@ mod tests {
|
||||
assert!(matches!(
|
||||
features.data.as_ref(),
|
||||
Some(MmKwargValue::Tensor(tensor))
|
||||
if tensor.dtype == "float32" && tensor.shape.first() == Some(&128)
|
||||
if tensor.dtype.as_str() == "float32" && tensor.shape.first() == Some(&128)
|
||||
));
|
||||
let lengths = &item.data["audio_feature_lengths"];
|
||||
assert!(matches!(&lengths.field, MmField::Batched(_)));
|
||||
assert!(matches!(
|
||||
lengths.data.as_ref(),
|
||||
Some(MmKwargValue::Tensor(tensor))
|
||||
if tensor.dtype == "int64" && tensor.shape.is_empty()
|
||||
if tensor.dtype.as_str() == "int64" && tensor.shape.is_empty()
|
||||
));
|
||||
}
|
||||
|
||||
@@ -314,14 +314,14 @@ mod tests {
|
||||
assert!(matches!(
|
||||
features.data.as_ref(),
|
||||
Some(MmKwargValue::Tensor(tensor))
|
||||
if tensor.dtype == "float32" && tensor.shape.get(1) == Some(&80)
|
||||
if tensor.dtype.as_str() == "float32" && tensor.shape.get(1) == Some(&80)
|
||||
));
|
||||
let count = &item.data["num_audio_tokens"];
|
||||
assert!(matches!(&count.field, MmField::Batched(_)));
|
||||
assert!(matches!(
|
||||
count.data.as_ref(),
|
||||
Some(MmKwargValue::Tensor(tensor))
|
||||
if tensor.dtype == "int64" && tensor.shape.is_empty()
|
||||
if tensor.dtype.as_str() == "int64" && tensor.shape.is_empty()
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -228,7 +228,7 @@ mod tests {
|
||||
|
||||
fn assert_bool_mask(range: &PlaceholderRange, expected: &[bool]) {
|
||||
let tensor = range.is_embed.as_ref().expect("is_embed mask");
|
||||
assert_eq!(tensor.dtype, "bool");
|
||||
assert_eq!(tensor.dtype.as_str(), "bool");
|
||||
assert_eq!(tensor.shape, vec![expected.len()]);
|
||||
assert_eq!(
|
||||
tensor.data,
|
||||
|
||||
@@ -7,54 +7,17 @@ use std::mem::size_of;
|
||||
use half::{bf16, f16};
|
||||
use llm_multimodal::{ModelSpecificValue, PreprocessedEncoderInputs};
|
||||
use ndarray::ArrayD;
|
||||
use vllm_engine_core_client::protocol::dtype::ModelDtype;
|
||||
use vllm_engine_core_client::protocol::dtype::{ModelDtype, TensorDtype};
|
||||
use vllm_engine_core_client::protocol::multimodal::MmKwargValue as ProtocolKwargValue;
|
||||
use vllm_engine_core_client::protocol::tensor::{ShapeExt as _, WireArrayData, WireTensor};
|
||||
use vllm_engine_core_client::protocol::tensor::{WireArrayData, WireTensor};
|
||||
|
||||
use crate::error::{Error, Result, bail_multimodal, multimodal};
|
||||
|
||||
/// Element type retained alongside an encoded tensor during multimodal lowering.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub(super) enum TensorKind {
|
||||
/// 32-bit floating point.
|
||||
F32,
|
||||
/// IEEE 16-bit floating point.
|
||||
F16,
|
||||
/// Brain floating point.
|
||||
Bf16,
|
||||
/// Signed 64-bit integer.
|
||||
I64,
|
||||
/// Unsigned 32-bit integer.
|
||||
U32,
|
||||
}
|
||||
|
||||
impl TensorKind {
|
||||
const fn element_size(self) -> usize {
|
||||
match self {
|
||||
Self::F32 => size_of::<f32>(),
|
||||
Self::F16 => size_of::<f16>(),
|
||||
Self::Bf16 => size_of::<bf16>(),
|
||||
Self::I64 => size_of::<i64>(),
|
||||
Self::U32 => size_of::<u32>(),
|
||||
}
|
||||
}
|
||||
|
||||
const fn wire_dtype(self) -> &'static str {
|
||||
match self {
|
||||
Self::F32 => "float32",
|
||||
Self::F16 => "float16",
|
||||
Self::Bf16 => "bfloat16",
|
||||
Self::I64 => "int64",
|
||||
Self::U32 => "uint32",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Representation for multimodal kwarg values for transformation.
|
||||
#[derive(Debug)]
|
||||
pub(super) enum KwargValue {
|
||||
/// Tensor with row-major flat data and shape.
|
||||
Tensor { kind: TensorKind, wire: WireTensor },
|
||||
Tensor { wire: WireTensor },
|
||||
/// Non-tensor kwarg value that is shared or copied as-is.
|
||||
Passthrough(ProtocolKwargValue),
|
||||
}
|
||||
@@ -96,11 +59,11 @@ impl KwargValue {
|
||||
}
|
||||
ModelSpecificValue::IntTensor { data, shape } => {
|
||||
let wire = WireTensor::from_i64(shape, data).map_err(Error::Multimodal)?;
|
||||
Self::tensor(TensorKind::I64, wire)
|
||||
Self::Tensor { wire }
|
||||
}
|
||||
ModelSpecificValue::UintTensor { data, shape } => {
|
||||
let wire = WireTensor::from_u32(shape, data).map_err(Error::Multimodal)?;
|
||||
Self::tensor(TensorKind::U32, wire)
|
||||
Self::Tensor { wire }
|
||||
}
|
||||
ModelSpecificValue::Int(value) => Self::Passthrough(Int(value)),
|
||||
ModelSpecificValue::Float(value) => Self::Passthrough(Float(value)),
|
||||
@@ -148,23 +111,16 @@ impl KwargValue {
|
||||
/// Convert a float tensor to the target float dtype if needed, keeping the
|
||||
/// same shape.
|
||||
fn from_f32_parts(data: Vec<f32>, shape: Vec<usize>, float_dtype: ModelDtype) -> Result<Self> {
|
||||
let (kind, wire) = match float_dtype {
|
||||
ModelDtype::Float16 => (
|
||||
TensorKind::F16,
|
||||
WireTensor::from_f16(shape, data.into_iter().map(f16::from_f32).collect()),
|
||||
),
|
||||
ModelDtype::BFloat16 => (
|
||||
TensorKind::Bf16,
|
||||
WireTensor::from_bf16(shape, data.into_iter().map(bf16::from_f32).collect()),
|
||||
),
|
||||
ModelDtype::Float32 => (TensorKind::F32, WireTensor::from_f32(shape, data)),
|
||||
let wire = match float_dtype {
|
||||
ModelDtype::Float16 => {
|
||||
WireTensor::from_f16(shape, data.into_iter().map(f16::from_f32).collect())
|
||||
}
|
||||
ModelDtype::BFloat16 => {
|
||||
WireTensor::from_bf16(shape, data.into_iter().map(bf16::from_f32).collect())
|
||||
}
|
||||
ModelDtype::Float32 => WireTensor::from_f32(shape, data),
|
||||
};
|
||||
wire.map(|wire| Self::tensor(kind, wire)).map_err(Error::Multimodal)
|
||||
}
|
||||
|
||||
fn tensor(kind: TensorKind, wire: WireTensor) -> Self {
|
||||
debug_assert_eq!(wire.dtype, kind.wire_dtype());
|
||||
Self::Tensor { kind, wire }
|
||||
wire.map(|wire| Self::Tensor { wire }).map_err(Error::Multimodal)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -215,9 +171,7 @@ impl KwargValue {
|
||||
drop_axis: bool,
|
||||
) -> Result<ProtocolKwargValue> {
|
||||
let wire = match self {
|
||||
Self::Tensor { kind, wire } => {
|
||||
slice_first_axis_range(wire, kind.element_size(), start, end, drop_axis)
|
||||
}
|
||||
Self::Tensor { wire } => slice_first_axis_range(wire, start, end, drop_axis),
|
||||
Self::Passthrough(value) => return Ok(value.clone()),
|
||||
};
|
||||
wire.map(ProtocolKwargValue::Tensor)
|
||||
@@ -244,10 +198,7 @@ pub(super) fn flat_range_for_index(
|
||||
/// Read a tensor value as per-image sizes for flat slicing.
|
||||
fn tensor_as_usize_vec(tensor: &KwargValue) -> Result<Vec<usize>> {
|
||||
match tensor {
|
||||
KwargValue::Tensor {
|
||||
kind: TensorKind::I64,
|
||||
wire,
|
||||
} => raw_tensor_bytes(wire, size_of::<i64>())?
|
||||
KwargValue::Tensor { wire } if wire.dtype == TensorDtype::I64 => raw_tensor_bytes(wire)?
|
||||
.chunks_exact(size_of::<i64>())
|
||||
.map(|bytes| i64::from_ne_bytes(bytes.try_into().expect("exact int64 chunk")))
|
||||
.map(|value| {
|
||||
@@ -255,13 +206,14 @@ fn tensor_as_usize_vec(tensor: &KwargValue) -> Result<Vec<usize>> {
|
||||
.map_err(|_| multimodal!("negative flat tensor size `{value}`"))
|
||||
})
|
||||
.collect(),
|
||||
KwargValue::Tensor {
|
||||
kind: TensorKind::U32,
|
||||
wire,
|
||||
} => Ok(raw_tensor_bytes(wire, size_of::<u32>())?
|
||||
.chunks_exact(size_of::<u32>())
|
||||
.map(|bytes| u32::from_ne_bytes(bytes.try_into().expect("exact uint32 chunk")) as usize)
|
||||
.collect()),
|
||||
KwargValue::Tensor { wire } if wire.dtype == TensorDtype::U32 => {
|
||||
Ok(raw_tensor_bytes(wire)?
|
||||
.chunks_exact(size_of::<u32>())
|
||||
.map(|bytes| {
|
||||
u32::from_ne_bytes(bytes.try_into().expect("exact uint32 chunk")) as usize
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
_ => Err(multimodal!("flat tensor sizes must be int64 or uint32")),
|
||||
}
|
||||
}
|
||||
@@ -269,13 +221,13 @@ fn tensor_as_usize_vec(tensor: &KwargValue) -> Result<Vec<usize>> {
|
||||
/// Slice a flat row-major tensor along its first axis.
|
||||
fn slice_first_axis_range(
|
||||
tensor: &WireTensor,
|
||||
element_size: usize,
|
||||
start: usize,
|
||||
end: usize,
|
||||
drop_axis: bool,
|
||||
) -> Result<WireTensor> {
|
||||
let shape = tensor.shape.as_slice();
|
||||
raw_tensor_bytes(tensor, element_size)?;
|
||||
raw_tensor_bytes(tensor)?;
|
||||
let element_size = tensor.dtype.element_size();
|
||||
let first_dim = *shape.first().ok_or_else(|| multimodal!("tensor has no first dimension"))?;
|
||||
if start > end || end > first_dim {
|
||||
bail_multimodal!("invalid tensor slice {start}..{end} for first dimension {first_dim}");
|
||||
@@ -302,34 +254,15 @@ fn slice_first_axis_range(
|
||||
return Err(multimodal!("cannot slice an aux tensor buffer"));
|
||||
};
|
||||
Ok(WireTensor::from_raw_bytes(
|
||||
tensor.dtype.clone(),
|
||||
tensor.dtype,
|
||||
out_shape,
|
||||
data.slice(data_start..data_end),
|
||||
))
|
||||
}
|
||||
|
||||
fn raw_tensor_bytes(tensor: &WireTensor, element_size: usize) -> Result<&[u8]> {
|
||||
let WireArrayData::RawView(data) = &tensor.data else {
|
||||
return Err(multimodal!("expected an inline tensor buffer"));
|
||||
};
|
||||
let expected_bytes = tensor
|
||||
.shape
|
||||
.checked_numel()
|
||||
.and_then(|numel| numel.checked_mul(element_size))
|
||||
.ok_or_else(|| {
|
||||
multimodal!(
|
||||
"tensor shape {:?} byte length overflowed usize",
|
||||
tensor.shape
|
||||
)
|
||||
})?;
|
||||
if expected_bytes != data.len() {
|
||||
bail_multimodal!(
|
||||
"tensor shape {:?} expects {expected_bytes} bytes, got {}",
|
||||
tensor.shape,
|
||||
data.len()
|
||||
);
|
||||
}
|
||||
Ok(data)
|
||||
fn raw_tensor_bytes(tensor: &WireTensor) -> Result<&[u8]> {
|
||||
tensor.validate_inline().map_err(Error::Multimodal)?;
|
||||
Ok(tensor.data.as_raw_view().expect("validated inline tensor"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -373,7 +306,7 @@ mod tests {
|
||||
};
|
||||
|
||||
assert_eq!(tensor.shape, shape, "{name} layout shape");
|
||||
assert_eq!(tensor.dtype, "float32", "{name} layout dtype");
|
||||
assert_eq!(tensor.dtype.as_str(), "float32", "{name} layout dtype");
|
||||
assert_eq!(
|
||||
tensor.data.into_raw_view().unwrap(),
|
||||
expected,
|
||||
@@ -386,10 +319,9 @@ mod tests {
|
||||
fn batched_wire_value_at_drops_first_axis() {
|
||||
let data = vec![1.0_f32, 2.0, 3.0, 4.0];
|
||||
let expected_ptr = data.as_ptr().cast::<u8>().wrapping_add(2 * size_of::<f32>());
|
||||
let value = KwargValue::tensor(
|
||||
TensorKind::F32,
|
||||
WireTensor::from_f32(vec![2, 2], data).unwrap(),
|
||||
);
|
||||
let value = KwargValue::Tensor {
|
||||
wire: WireTensor::from_f32(vec![2, 2], data).unwrap(),
|
||||
};
|
||||
|
||||
let ProtocolKwargValue::Tensor(tensor) = value.batched_wire_value_at(1).unwrap() else {
|
||||
panic!("expected tensor");
|
||||
@@ -406,10 +338,9 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn flat_wire_value_range_keeps_first_axis() {
|
||||
let value = KwargValue::tensor(
|
||||
TensorKind::U32,
|
||||
WireTensor::from_u32(vec![5, 2], (0..10_u32).collect()).unwrap(),
|
||||
);
|
||||
let value = KwargValue::Tensor {
|
||||
wire: WireTensor::from_u32(vec![5, 2], (0..10_u32).collect()).unwrap(),
|
||||
};
|
||||
|
||||
let ProtocolKwargValue::Tensor(tensor) = value.flat_wire_value_range(1, 3).unwrap() else {
|
||||
panic!("expected tensor");
|
||||
@@ -424,10 +355,9 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn flat_range_for_index_uses_size_tensor() {
|
||||
let sizes = KwargValue::tensor(
|
||||
TensorKind::I64,
|
||||
WireTensor::from_i64(vec![3], vec![2_i64, 3, 4]).unwrap(),
|
||||
);
|
||||
let sizes = KwargValue::Tensor {
|
||||
wire: WireTensor::from_i64(vec![3], vec![2_i64, 3, 4]).unwrap(),
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
flat_range_for_index(&sizes, "image_grid_thw", 1).unwrap(),
|
||||
@@ -437,11 +367,12 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn slice_first_axis_range_errors_on_shape_data_mismatch() {
|
||||
let tensor = WireTensor::from_raw("float32", vec![2, 2], vec![0; 3 * size_of::<f32>()]);
|
||||
let error = slice_first_axis_range(&tensor, size_of::<f32>(), 0, 1, true).unwrap_err();
|
||||
let tensor =
|
||||
WireTensor::from_raw(TensorDtype::F32, vec![2, 2], vec![0; 3 * size_of::<f32>()]);
|
||||
let error = slice_first_axis_range(&tensor, 0, 1, true).unwrap_err();
|
||||
|
||||
assert!(
|
||||
matches!(error, Error::Multimodal(message) if message.contains("expects 16 bytes"))
|
||||
matches!(error, Error::Multimodal(message) if message.contains("does not match expected 16"))
|
||||
);
|
||||
}
|
||||
|
||||
@@ -455,7 +386,7 @@ mod tests {
|
||||
panic!("expected tensor");
|
||||
};
|
||||
|
||||
assert_eq!(tensor.dtype, "bfloat16");
|
||||
assert_eq!(tensor.dtype.as_str(), "bfloat16");
|
||||
assert_eq!(tensor.shape, vec![2]);
|
||||
assert_eq!(tensor.data.into_raw_view().unwrap().len(), 4);
|
||||
}
|
||||
@@ -470,7 +401,7 @@ mod tests {
|
||||
panic!("expected tensor");
|
||||
};
|
||||
|
||||
assert_eq!(tensor.dtype, "float16");
|
||||
assert_eq!(tensor.dtype.as_str(), "float16");
|
||||
assert_eq!(tensor.shape, vec![2]);
|
||||
assert_eq!(tensor.data.into_raw_view().unwrap().len(), 4);
|
||||
}
|
||||
|
||||
@@ -1,7 +1,191 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::fmt;
|
||||
use std::str::FromStr;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_with::{DeserializeFromStr, SerializeDisplay};
|
||||
|
||||
/// Element types accepted by Python's tensor decoder.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, SerializeDisplay, DeserializeFromStr)]
|
||||
pub enum TensorDtype {
|
||||
Bool,
|
||||
U8,
|
||||
I8,
|
||||
U16,
|
||||
I16,
|
||||
U32,
|
||||
I32,
|
||||
U64,
|
||||
I64,
|
||||
F16,
|
||||
Bf16,
|
||||
F32,
|
||||
F64,
|
||||
}
|
||||
|
||||
impl TensorDtype {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Bool => "bool",
|
||||
Self::U8 => "uint8",
|
||||
Self::I8 => "int8",
|
||||
Self::U16 => "uint16",
|
||||
Self::I16 => "int16",
|
||||
Self::U32 => "uint32",
|
||||
Self::I32 => "int32",
|
||||
Self::U64 => "uint64",
|
||||
Self::I64 => "int64",
|
||||
Self::F16 => "float16",
|
||||
Self::Bf16 => "bfloat16",
|
||||
Self::F32 => "float32",
|
||||
Self::F64 => "float64",
|
||||
}
|
||||
}
|
||||
|
||||
pub const fn element_size(self) -> usize {
|
||||
match self {
|
||||
Self::Bool | Self::U8 | Self::I8 => 1,
|
||||
Self::U16 | Self::I16 | Self::F16 | Self::Bf16 => 2,
|
||||
Self::U32 | Self::I32 | Self::F32 => 4,
|
||||
Self::U64 | Self::I64 | Self::F64 => 8,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for TensorDtype {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for TensorDtype {
|
||||
type Err = String;
|
||||
|
||||
fn from_str(value: &str) -> Result<Self, Self::Err> {
|
||||
match value {
|
||||
"bool" => Ok(Self::Bool),
|
||||
"uint8" => Ok(Self::U8),
|
||||
"int8" => Ok(Self::I8),
|
||||
"uint16" => Ok(Self::U16),
|
||||
"int16" => Ok(Self::I16),
|
||||
"uint32" => Ok(Self::U32),
|
||||
"int32" => Ok(Self::I32),
|
||||
"uint64" => Ok(Self::U64),
|
||||
"int64" => Ok(Self::I64),
|
||||
"float16" => Ok(Self::F16),
|
||||
"bfloat16" => Ok(Self::Bf16),
|
||||
"float32" => Ok(Self::F32),
|
||||
"float64" => Ok(Self::F64),
|
||||
_ => Err(format!("unsupported tensor dtype {value:?}")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Byte order carried by a NumPy dtype descriptor.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Endianness {
|
||||
Little,
|
||||
Big,
|
||||
Native,
|
||||
}
|
||||
|
||||
/// Numeric array dtype with an explicit byte-order contract.
|
||||
///
|
||||
/// Also accepts Torch type names as native-endian dtypes because prompt
|
||||
/// logprobs use tensor tuples with the same wire shape as NumPy arrays.
|
||||
/// Serializes byte order explicitly, including `=` for native byte order.
|
||||
/// Bfloat16 uses the Torch name after the prefix as a nonstandard descriptor.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, SerializeDisplay, DeserializeFromStr)]
|
||||
pub struct NumpyDtype {
|
||||
pub scalar: TensorDtype,
|
||||
pub endianness: Endianness,
|
||||
}
|
||||
|
||||
impl NumpyDtype {
|
||||
pub const fn little(scalar: TensorDtype) -> Self {
|
||||
Self {
|
||||
scalar,
|
||||
endianness: Endianness::Little,
|
||||
}
|
||||
}
|
||||
|
||||
pub const fn native(scalar: TensorDtype) -> Self {
|
||||
Self {
|
||||
scalar,
|
||||
endianness: Endianness::Native,
|
||||
}
|
||||
}
|
||||
|
||||
pub const fn big(scalar: TensorDtype) -> Self {
|
||||
Self {
|
||||
scalar,
|
||||
endianness: Endianness::Big,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<TensorDtype> for NumpyDtype {
|
||||
fn from(scalar: TensorDtype) -> Self {
|
||||
Self::native(scalar)
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for NumpyDtype {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
let prefix = match self.endianness {
|
||||
Endianness::Native => '=',
|
||||
Endianness::Little => '<',
|
||||
Endianness::Big => '>',
|
||||
};
|
||||
let code = match self.scalar {
|
||||
TensorDtype::Bool => "b1",
|
||||
TensorDtype::U8 => "u1",
|
||||
TensorDtype::I8 => "i1",
|
||||
TensorDtype::U16 => "u2",
|
||||
TensorDtype::I16 => "i2",
|
||||
TensorDtype::U32 => "u4",
|
||||
TensorDtype::I32 => "i4",
|
||||
TensorDtype::U64 => "u8",
|
||||
TensorDtype::I64 => "i8",
|
||||
TensorDtype::F16 => "f2",
|
||||
TensorDtype::F32 => "f4",
|
||||
TensorDtype::F64 => "f8",
|
||||
TensorDtype::Bf16 => "bfloat16",
|
||||
};
|
||||
write!(f, "{prefix}{code}")
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for NumpyDtype {
|
||||
type Err = String;
|
||||
|
||||
fn from_str(dtype: &str) -> Result<Self, Self::Err> {
|
||||
let (endianness, body) = match dtype.as_bytes().first().copied() {
|
||||
Some(b'<') => (Endianness::Little, &dtype[1..]),
|
||||
Some(b'>') => (Endianness::Big, &dtype[1..]),
|
||||
Some(b'=' | b'|') => (Endianness::Native, &dtype[1..]),
|
||||
_ => (Endianness::Native, dtype),
|
||||
};
|
||||
let scalar = match body {
|
||||
"b1" => TensorDtype::Bool,
|
||||
"u1" => TensorDtype::U8,
|
||||
"i1" => TensorDtype::I8,
|
||||
"u2" => TensorDtype::U16,
|
||||
"i2" => TensorDtype::I16,
|
||||
"u4" => TensorDtype::U32,
|
||||
"i4" => TensorDtype::I32,
|
||||
"u8" => TensorDtype::U64,
|
||||
"i8" => TensorDtype::I64,
|
||||
"f2" => TensorDtype::F16,
|
||||
"f4" => TensorDtype::F32,
|
||||
"f8" => TensorDtype::F64,
|
||||
_ => body.parse().map_err(|_| format!("unsupported dtype string {dtype:?}"))?,
|
||||
};
|
||||
Ok(Self { scalar, endianness })
|
||||
}
|
||||
}
|
||||
|
||||
/// Effective model dtype reported by the engine after config resolution.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
@@ -26,7 +210,53 @@ impl ModelDtype {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::ModelDtype;
|
||||
use super::{Endianness, ModelDtype, NumpyDtype, TensorDtype};
|
||||
|
||||
#[test]
|
||||
fn numpy_descriptors_preserve_explicit_byte_order() {
|
||||
for (code, scalar) in [
|
||||
("b1", TensorDtype::Bool),
|
||||
("u1", TensorDtype::U8),
|
||||
("i1", TensorDtype::I8),
|
||||
("u2", TensorDtype::U16),
|
||||
("i2", TensorDtype::I16),
|
||||
("u4", TensorDtype::U32),
|
||||
("i4", TensorDtype::I32),
|
||||
("u8", TensorDtype::U64),
|
||||
("i8", TensorDtype::I64),
|
||||
("f2", TensorDtype::F16),
|
||||
("bfloat16", TensorDtype::Bf16),
|
||||
("f4", TensorDtype::F32),
|
||||
("f8", TensorDtype::F64),
|
||||
] {
|
||||
for (prefix, endianness) in [("<", Endianness::Little), (">", Endianness::Big)] {
|
||||
let wire = serde_json::json!(format!("{prefix}{code}"));
|
||||
let dtype: NumpyDtype = serde_json::from_value(wire.clone()).unwrap();
|
||||
assert_eq!(dtype, NumpyDtype { scalar, endianness });
|
||||
assert_eq!(serde_json::to_value(dtype).unwrap(), wire);
|
||||
}
|
||||
for wire in [
|
||||
format!("={code}"),
|
||||
format!("|{code}"),
|
||||
code.into(),
|
||||
scalar.as_str().into(),
|
||||
] {
|
||||
let dtype: NumpyDtype = serde_json::from_value(serde_json::json!(wire)).unwrap();
|
||||
assert_eq!(dtype, NumpyDtype::from(scalar));
|
||||
assert_eq!(
|
||||
serde_json::to_value(dtype).unwrap(),
|
||||
serde_json::json!(format!("={code}"))
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_unsupported_numpy_dtypes() {
|
||||
for wire in ["", "object", "<c8", "<bf2"] {
|
||||
assert!(serde_json::from_value::<NumpyDtype>(serde_json::json!(wire)).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serde_uses_protocol_dtype_strings() {
|
||||
|
||||
@@ -14,6 +14,7 @@ use serde::{Deserialize, Deserializer, Serialize};
|
||||
|
||||
use self::wire::*;
|
||||
use crate::error::{Error, Result, bail_ext_value_decode};
|
||||
use crate::protocol::dtype::{NumpyDtype, TensorDtype};
|
||||
use crate::protocol::tensor::{WireArrayData, WireNdArray};
|
||||
|
||||
/// One token candidate and its logprob metadata for a single sequence position.
|
||||
@@ -205,17 +206,17 @@ impl WireLogprobs {
|
||||
|
||||
Ok(Self {
|
||||
logprob_token_ids: WireNdArray {
|
||||
dtype: "<i8".to_string(),
|
||||
dtype: NumpyDtype::little(TensorDtype::I64),
|
||||
shape: vec![rows, cols],
|
||||
data: WireArrayData::RawView(token_ids.into()),
|
||||
},
|
||||
logprobs: WireNdArray {
|
||||
dtype: "<f4".to_string(),
|
||||
dtype: NumpyDtype::little(TensorDtype::F32),
|
||||
shape: vec![rows, cols],
|
||||
data: WireArrayData::RawView(logprobs.into()),
|
||||
},
|
||||
token_ranks: WireNdArray {
|
||||
dtype: "<i8".to_string(),
|
||||
dtype: NumpyDtype::little(TensorDtype::I64),
|
||||
shape: vec![rows],
|
||||
data: WireArrayData::RawView(token_ranks.into()),
|
||||
},
|
||||
|
||||
@@ -8,22 +8,9 @@ use bytes::Bytes;
|
||||
use itertools::Itertools as _;
|
||||
|
||||
use crate::error::{Error, Result, ext_value_decode};
|
||||
use crate::protocol::dtype::{Endianness, TensorDtype};
|
||||
use crate::protocol::tensor::{ShapeExt as _, WireNdArray};
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum ScalarType {
|
||||
I32,
|
||||
I64,
|
||||
F32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum Endianness {
|
||||
Little,
|
||||
Big,
|
||||
Native,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub(super) struct DecodedArray2<T> {
|
||||
pub rows: usize,
|
||||
@@ -37,7 +24,7 @@ pub(super) fn decode_array2_u32(
|
||||
frames: &[Bytes],
|
||||
) -> Result<DecodedArray2<u32>> {
|
||||
let (shape, bytes, scalar, endianness) =
|
||||
decode_array_metadata(value, field, frames, &[ScalarType::I32, ScalarType::I64])?;
|
||||
decode_array_metadata(value, field, frames, &[TensorDtype::I32, TensorDtype::I64])?;
|
||||
if shape.len() != 2 {
|
||||
return Err(decode_error(
|
||||
field,
|
||||
@@ -46,15 +33,15 @@ pub(super) fn decode_array2_u32(
|
||||
}
|
||||
|
||||
let data = match scalar {
|
||||
ScalarType::I32 => decode_i32_vec(&bytes, endianness, field)?
|
||||
TensorDtype::I32 => decode_i32_vec(&bytes, endianness, field)?
|
||||
.into_iter()
|
||||
.map(|value| convert_to_u32(value, field))
|
||||
.try_collect()?,
|
||||
ScalarType::I64 => decode_i64_vec(&bytes, endianness, field)?
|
||||
TensorDtype::I64 => decode_i64_vec(&bytes, endianness, field)?
|
||||
.into_iter()
|
||||
.map(|value| convert_to_u32(value, field))
|
||||
.try_collect()?,
|
||||
ScalarType::F32 => unreachable!("scalar validation should reject f32"),
|
||||
_ => unreachable!("scalar validation should accept only i32 and i64"),
|
||||
};
|
||||
Ok(DecodedArray2 {
|
||||
rows: shape[0],
|
||||
@@ -69,7 +56,7 @@ pub(super) fn decode_array1_u32(
|
||||
frames: &[Bytes],
|
||||
) -> Result<Vec<u32>> {
|
||||
let (shape, bytes, scalar, endianness) =
|
||||
decode_array_metadata(value, field, frames, &[ScalarType::I32, ScalarType::I64])?;
|
||||
decode_array_metadata(value, field, frames, &[TensorDtype::I32, TensorDtype::I64])?;
|
||||
if shape.len() != 1 {
|
||||
return Err(decode_error(
|
||||
field,
|
||||
@@ -78,15 +65,15 @@ pub(super) fn decode_array1_u32(
|
||||
}
|
||||
|
||||
let data = match scalar {
|
||||
ScalarType::I32 => decode_i32_vec(&bytes, endianness, field)?
|
||||
TensorDtype::I32 => decode_i32_vec(&bytes, endianness, field)?
|
||||
.into_iter()
|
||||
.map(|value| convert_to_u32(value, field))
|
||||
.try_collect()?,
|
||||
ScalarType::I64 => decode_i64_vec(&bytes, endianness, field)?
|
||||
TensorDtype::I64 => decode_i64_vec(&bytes, endianness, field)?
|
||||
.into_iter()
|
||||
.map(|value| convert_to_u32(value, field))
|
||||
.try_collect()?,
|
||||
ScalarType::F32 => unreachable!("scalar validation should reject f32"),
|
||||
_ => unreachable!("scalar validation should accept only i32 and i64"),
|
||||
};
|
||||
Ok(data)
|
||||
}
|
||||
@@ -97,7 +84,7 @@ pub(super) fn decode_array2_f32(
|
||||
frames: &[Bytes],
|
||||
) -> Result<DecodedArray2<f32>> {
|
||||
let (shape, bytes, _, endianness) =
|
||||
decode_array_metadata(value, field, frames, &[ScalarType::F32])?;
|
||||
decode_array_metadata(value, field, frames, &[TensorDtype::F32])?;
|
||||
if shape.len() != 2 {
|
||||
return Err(decode_error(
|
||||
field,
|
||||
@@ -117,15 +104,16 @@ pub(super) fn decode_array_metadata(
|
||||
value: WireNdArray,
|
||||
field: &str,
|
||||
frames: &[Bytes],
|
||||
expected_scalars: &[ScalarType],
|
||||
) -> Result<(Vec<usize>, Bytes, ScalarType, Endianness)> {
|
||||
expected_scalars: &[TensorDtype],
|
||||
) -> Result<(Vec<usize>, Bytes, TensorDtype, Endianness)> {
|
||||
let mut value = value;
|
||||
let (scalar, endianness) = parse_dtype(&value.dtype, field)?;
|
||||
let scalar = value.dtype.scalar;
|
||||
let endianness = value.dtype.endianness;
|
||||
if !expected_scalars.contains(&scalar) {
|
||||
return Err(decode_error(
|
||||
field,
|
||||
&format!(
|
||||
"expected dtype in {:?}, got {}",
|
||||
"expected dtype in {:?}, got {:?}",
|
||||
expected_scalars, value.dtype
|
||||
),
|
||||
));
|
||||
@@ -140,42 +128,16 @@ pub(super) fn decode_array_metadata(
|
||||
Ok((shape, bytes, scalar, endianness))
|
||||
}
|
||||
|
||||
pub(super) fn parse_dtype(dtype: &str, field: &str) -> Result<(ScalarType, Endianness)> {
|
||||
let (endianness, body) = match dtype.as_bytes().first().copied() {
|
||||
Some(b'<') => (Endianness::Little, &dtype[1..]),
|
||||
Some(b'>') => (Endianness::Big, &dtype[1..]),
|
||||
Some(b'=') => (Endianness::Native, &dtype[1..]),
|
||||
Some(b'|') => (Endianness::Native, &dtype[1..]),
|
||||
_ => (Endianness::Native, dtype),
|
||||
};
|
||||
|
||||
let scalar = match body {
|
||||
"i4" | "int32" => ScalarType::I32,
|
||||
"i8" | "int64" => ScalarType::I64,
|
||||
"f4" | "float32" => ScalarType::F32,
|
||||
_ => {
|
||||
return Err(decode_error(
|
||||
field,
|
||||
&format!("unsupported dtype string {dtype:?}"),
|
||||
));
|
||||
}
|
||||
};
|
||||
Ok((scalar, endianness))
|
||||
}
|
||||
|
||||
pub(super) fn validate_byte_length(
|
||||
shape: &[usize],
|
||||
byte_len: usize,
|
||||
field: &str,
|
||||
scalar: ScalarType,
|
||||
scalar: TensorDtype,
|
||||
) -> Result<()> {
|
||||
let element_count = shape
|
||||
.checked_numel()
|
||||
.ok_or_else(|| decode_error(field, "shape element count overflowed usize"))?;
|
||||
let element_size = match scalar {
|
||||
ScalarType::I32 | ScalarType::F32 => 4,
|
||||
ScalarType::I64 => 8,
|
||||
};
|
||||
let element_size = scalar.element_size();
|
||||
let expected = element_count
|
||||
.checked_mul(element_size)
|
||||
.ok_or_else(|| decode_error(field, "byte length overflowed usize"))?;
|
||||
|
||||
@@ -302,6 +302,29 @@ fn decodes_big_endian_payloads() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_supported_array_dtypes_in_incompatible_logprobs_fields() {
|
||||
for (ids_dtype, probs_dtype, field) in [
|
||||
("<u4", "<f4", "logprob_token_ids"),
|
||||
("<i4", "<i4", "logprobs"),
|
||||
] {
|
||||
let frames = vec![Bytes::from(encode_value(&output_wire_with_custom_fields(
|
||||
Some(Value::Array(vec![
|
||||
ndarray_value(ids_dtype, &[1, 1], Value::Ext(3, vec![0; 4])),
|
||||
ndarray_value(probs_dtype, &[1, 1], Value::Ext(3, vec![0; 4])),
|
||||
ndarray_value("<i4", &[1], Value::Ext(3, vec![1, 0, 0, 0])),
|
||||
Value::Nil,
|
||||
])),
|
||||
None,
|
||||
)))];
|
||||
let error = decode_engine_core_outputs(&frames).unwrap_err();
|
||||
let crate::error::Error::ExtValueDecode { message } = error else {
|
||||
panic!("expected ExtValueDecode");
|
||||
};
|
||||
assert!(message.starts_with(&format!("new_logprobs.{field}: expected dtype")));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_non_none_cu_num_generated_tokens() {
|
||||
let frames = vec![Bytes::from(encode_value(&output_wire_with_custom_fields(
|
||||
|
||||
@@ -9,6 +9,26 @@ use serde_tuple::{Deserialize_tuple, Serialize_tuple};
|
||||
|
||||
use super::tensor::WireTensor;
|
||||
|
||||
/// Modalities represented by engine-ready multimodal features.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum MmModality {
|
||||
Image,
|
||||
Audio,
|
||||
Video,
|
||||
}
|
||||
|
||||
impl MmModality {
|
||||
/// The wire name, matching Python's modality strings.
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Image => "image",
|
||||
Self::Audio => "audio",
|
||||
Self::Video => "video",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Multimodal feature payload accepted from higher-level frontend code.
|
||||
///
|
||||
/// Original Python definition:
|
||||
@@ -32,7 +52,7 @@ pub struct MmFeatureSpec {
|
||||
pub data: Option<MmKwargsItem>,
|
||||
|
||||
/// The input modality, e.g., `"image"`, `"audio"`, `"video"`.
|
||||
pub modality: String,
|
||||
pub modality: MmModality,
|
||||
|
||||
/// The hash for caching encoder outputs (with LoRA prefix if applicable).
|
||||
pub identifier: String,
|
||||
|
||||
@@ -158,8 +158,10 @@ mod tests {
|
||||
use rmpv::Value;
|
||||
|
||||
use super::*;
|
||||
use crate::protocol::dtype::TensorDtype;
|
||||
use crate::protocol::multimodal::{
|
||||
MmBatchedField, MmFeatureSpec, MmField, MmFieldElem, MmKwargValue, PlaceholderRange,
|
||||
MmBatchedField, MmFeatureSpec, MmField, MmFieldElem, MmKwargValue, MmModality,
|
||||
PlaceholderRange,
|
||||
};
|
||||
use crate::protocol::sampling::EngineCoreSamplingParams;
|
||||
use crate::protocol::tensor::{WireArrayData, WireTensor};
|
||||
@@ -212,7 +214,7 @@ mod tests {
|
||||
"inline".to_string(),
|
||||
MmFieldElem {
|
||||
data: Some(MmKwargValue::Tensor(WireTensor::from_raw(
|
||||
"uint8",
|
||||
TensorDtype::U8,
|
||||
vec![inline.len()],
|
||||
inline,
|
||||
))),
|
||||
@@ -225,7 +227,7 @@ mod tests {
|
||||
data: Some(MmKwargValue::List(vec![
|
||||
MmKwargValue::Int(7),
|
||||
MmKwargValue::Tensor(WireTensor::from_raw(
|
||||
"uint8",
|
||||
TensorDtype::U8,
|
||||
vec![first_aux.len()],
|
||||
first_aux,
|
||||
)),
|
||||
@@ -234,13 +236,13 @@ mod tests {
|
||||
},
|
||||
),
|
||||
])),
|
||||
modality: "image".to_string(),
|
||||
modality: MmModality::Image,
|
||||
identifier: "id".to_string(),
|
||||
mm_position: PlaceholderRange {
|
||||
offset: 0,
|
||||
length: second_aux.len(),
|
||||
is_embed: Some(WireTensor::from_raw(
|
||||
"bool",
|
||||
TensorDtype::Bool,
|
||||
vec![second_aux.len()],
|
||||
second_aux,
|
||||
)),
|
||||
|
||||
@@ -6,9 +6,12 @@ use bytes::Bytes;
|
||||
use enum_as_inner::EnumAsInner;
|
||||
use half::{bf16, f16};
|
||||
use rmpv::Value;
|
||||
use serde::de::DeserializeOwned;
|
||||
use serde::{Deserialize, Deserializer, Serialize, Serializer};
|
||||
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
|
||||
|
||||
use super::dtype::{NumpyDtype, TensorDtype};
|
||||
|
||||
/// Tensors and ndarrays are encoded with this extension type in Python.
|
||||
///
|
||||
/// Original Python definition:
|
||||
@@ -63,20 +66,20 @@ impl [usize] {
|
||||
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/v1/serial_utils.py#L237-L273>
|
||||
/// <https://github.com/vllm-project/vllm/blob/5a0a8fc1ea7542394ff315138bd5677b7b53bca1/vllm/v1/serial_utils.py#L389-L425>
|
||||
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple)]
|
||||
pub struct WireNdArray {
|
||||
pub dtype: String,
|
||||
pub struct WireArray<Dtype: Serialize + DeserializeOwned> {
|
||||
pub dtype: Dtype,
|
||||
pub shape: Vec<usize>,
|
||||
pub data: WireArrayData,
|
||||
}
|
||||
|
||||
impl WireNdArray {
|
||||
impl<Dtype: From<TensorDtype> + Serialize + DeserializeOwned> WireArray<Dtype> {
|
||||
/// Build a float32 tensor/ndarray backed by native-endian raw-view bytes.
|
||||
///
|
||||
/// Takes ownership of the backing buffer without copying its data.
|
||||
pub fn from_f32(shape: Vec<usize>, data: Vec<f32>) -> Result<Self, String> {
|
||||
validate_element_count(&shape, data.len())?;
|
||||
Ok(Self::from_raw_bytes(
|
||||
"float32",
|
||||
TensorDtype::F32,
|
||||
shape,
|
||||
bytes_from_pod_vec(data),
|
||||
))
|
||||
@@ -88,7 +91,7 @@ impl WireNdArray {
|
||||
pub fn from_f16(shape: Vec<usize>, data: Vec<f16>) -> Result<Self, String> {
|
||||
validate_element_count(&shape, data.len())?;
|
||||
Ok(Self::from_raw_bytes(
|
||||
"float16",
|
||||
TensorDtype::F16,
|
||||
shape,
|
||||
bytes_from_pod_vec(data),
|
||||
))
|
||||
@@ -100,7 +103,7 @@ impl WireNdArray {
|
||||
pub fn from_bf16(shape: Vec<usize>, data: Vec<bf16>) -> Result<Self, String> {
|
||||
validate_element_count(&shape, data.len())?;
|
||||
Ok(Self::from_raw_bytes(
|
||||
"bfloat16",
|
||||
TensorDtype::Bf16,
|
||||
shape,
|
||||
bytes_from_pod_vec(data),
|
||||
))
|
||||
@@ -112,7 +115,7 @@ impl WireNdArray {
|
||||
pub fn from_i64(shape: Vec<usize>, data: Vec<i64>) -> Result<Self, String> {
|
||||
validate_element_count(&shape, data.len())?;
|
||||
Ok(Self::from_raw_bytes(
|
||||
"int64",
|
||||
TensorDtype::I64,
|
||||
shape,
|
||||
bytes_from_pod_vec(data),
|
||||
))
|
||||
@@ -124,7 +127,7 @@ impl WireNdArray {
|
||||
pub fn from_u32(shape: Vec<usize>, data: Vec<u32>) -> Result<Self, String> {
|
||||
validate_element_count(&shape, data.len())?;
|
||||
Ok(Self::from_raw_bytes(
|
||||
"uint32",
|
||||
TensorDtype::U32,
|
||||
shape,
|
||||
bytes_from_pod_vec(data),
|
||||
))
|
||||
@@ -137,7 +140,7 @@ impl WireNdArray {
|
||||
pub fn from_bool(shape: Vec<usize>, data: Vec<bool>) -> Result<Self, String> {
|
||||
validate_element_count(&shape, data.len())?;
|
||||
Ok(Self {
|
||||
dtype: "bool".to_string(),
|
||||
dtype: TensorDtype::Bool.into(),
|
||||
shape,
|
||||
data: WireArrayData::RawView(Bytes::from(
|
||||
data.into_iter().map(u8::from).collect::<Vec<_>>(),
|
||||
@@ -149,12 +152,12 @@ impl WireNdArray {
|
||||
///
|
||||
/// Use this as an escape hatch when the caller already owns bytes that
|
||||
/// match the requested `dtype` and `shape`.
|
||||
pub fn from_raw(dtype: impl Into<String>, shape: Vec<usize>, data: Vec<u8>) -> Self {
|
||||
pub fn from_raw(dtype: impl Into<Dtype>, shape: Vec<usize>, data: Vec<u8>) -> Self {
|
||||
Self::from_raw_bytes(dtype, shape, Bytes::from(data))
|
||||
}
|
||||
|
||||
/// Build a tensor/ndarray from an owned immutable raw-view buffer.
|
||||
pub fn from_raw_bytes(dtype: impl Into<String>, shape: Vec<usize>, data: Bytes) -> Self {
|
||||
pub fn from_raw_bytes(dtype: impl Into<Dtype>, shape: Vec<usize>, data: Bytes) -> Self {
|
||||
Self {
|
||||
dtype: dtype.into(),
|
||||
shape,
|
||||
@@ -192,7 +195,10 @@ fn validate_element_count(shape: &[usize], len: usize) -> Result<(), String> {
|
||||
///
|
||||
/// This is the same wire shape as [`WireNdArray`]; multimodal request payloads
|
||||
/// use it for `torch.Tensor` values.
|
||||
pub type WireTensor = WireNdArray;
|
||||
pub type WireTensor = WireArray<TensorDtype>;
|
||||
|
||||
/// Numeric array with a typed NumPy descriptor or native-endian Torch dtype.
|
||||
pub type WireNdArray = WireArray<NumpyDtype>;
|
||||
|
||||
/// Python array/tensor payload reference inside [`WireNdArray`].
|
||||
///
|
||||
@@ -211,6 +217,27 @@ pub enum WireArrayData {
|
||||
RawView(Bytes),
|
||||
}
|
||||
|
||||
impl WireTensor {
|
||||
/// Validate the shape and storage of an inline tensor.
|
||||
pub fn validate_inline(&self) -> Result<(), String> {
|
||||
let expected = self
|
||||
.shape
|
||||
.checked_numel()
|
||||
.and_then(|numel| numel.checked_mul(self.dtype.element_size()))
|
||||
.ok_or_else(|| format!("tensor shape or byte length overflows: {:?}", self.shape))?;
|
||||
match &self.data {
|
||||
WireArrayData::RawView(bytes) if bytes.len() == expected => Ok(()),
|
||||
WireArrayData::RawView(bytes) => Err(format!(
|
||||
"tensor byte length {} does not match expected {expected}",
|
||||
bytes.len()
|
||||
)),
|
||||
WireArrayData::AuxIndex(index) => {
|
||||
Err(format!("inline tensor references auxiliary frame {index}"))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl WireArrayData {
|
||||
/// Resolve an auxiliary-frame reference into owned raw bytes.
|
||||
pub(crate) fn resolve_aux_frame(&mut self, frames: &[Bytes]) -> Result<(), String> {
|
||||
@@ -288,6 +315,21 @@ impl Serialize for WireArrayData {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn tensor_and_ndarray_preserve_their_dtype_wire_contracts() {
|
||||
let tensor = WireTensor::from_f32(vec![1], vec![1.0]).unwrap();
|
||||
let bytes = rmp_serde::to_vec_named(&tensor).unwrap();
|
||||
let wire: Value = rmp_serde::from_slice(&bytes).unwrap();
|
||||
assert_eq!(wire.as_array().unwrap()[0], Value::from("float32"));
|
||||
|
||||
let array: WireNdArray = rmp_serde::from_slice(&bytes).unwrap();
|
||||
assert_eq!(array.dtype, NumpyDtype::native(TensorDtype::F32));
|
||||
let bytes = rmp_serde::to_vec_named(&array).unwrap();
|
||||
let wire: Value = rmp_serde::from_slice(&bytes).unwrap();
|
||||
assert_eq!(wire.as_array().unwrap()[0], Value::from("=f4"));
|
||||
assert!(rmp_serde::from_slice::<WireTensor>(&bytes).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn raw_view_serializes_as_msgpack_ext() {
|
||||
let bytes = vec![1, 2, 3, 4];
|
||||
@@ -307,8 +349,8 @@ mod tests {
|
||||
fn constructors_build_raw_view_tensors() {
|
||||
let f32_data = vec![1.0, 2.5];
|
||||
let f32_data_ptr = f32_data.as_ptr().cast::<u8>();
|
||||
let f32_tensor = WireNdArray::from_f32(vec![2], f32_data).unwrap();
|
||||
assert_eq!(f32_tensor.dtype, "float32");
|
||||
let f32_tensor = WireTensor::from_f32(vec![2], f32_data).unwrap();
|
||||
assert_eq!(f32_tensor.dtype, TensorDtype::F32);
|
||||
assert_eq!(f32_tensor.shape, vec![2]);
|
||||
let f32_raw_view = f32_tensor.data.into_raw_view().expect("raw view");
|
||||
assert_eq!(f32_raw_view.as_ptr(), f32_data_ptr);
|
||||
@@ -318,58 +360,49 @@ mod tests {
|
||||
);
|
||||
|
||||
let f16_tensor =
|
||||
WireNdArray::from_f16(vec![2], vec![f16::from_f32(1.0), f16::from_f32(2.5)]).unwrap();
|
||||
assert_eq!(f16_tensor.dtype, "float16");
|
||||
WireTensor::from_f16(vec![2], vec![f16::from_f32(1.0), f16::from_f32(2.5)]).unwrap();
|
||||
assert_eq!(f16_tensor.dtype, TensorDtype::F16);
|
||||
assert_eq!(f16_tensor.shape, vec![2]);
|
||||
assert_eq!(f16_tensor.data.into_raw_view().expect("raw view").len(), 4);
|
||||
|
||||
let bf16_tensor =
|
||||
WireNdArray::from_bf16(vec![2], vec![bf16::from_f32(1.0), bf16::from_f32(2.5)])
|
||||
.unwrap();
|
||||
assert_eq!(bf16_tensor.dtype, "bfloat16");
|
||||
WireTensor::from_bf16(vec![2], vec![bf16::from_f32(1.0), bf16::from_f32(2.5)]).unwrap();
|
||||
assert_eq!(bf16_tensor.dtype, TensorDtype::Bf16);
|
||||
assert_eq!(bf16_tensor.shape, vec![2]);
|
||||
assert_eq!(bf16_tensor.data.into_raw_view().expect("raw view").len(), 4);
|
||||
|
||||
let i64_tensor = WireNdArray::from_i64(vec![1], vec![-7]).unwrap();
|
||||
assert_eq!(i64_tensor.dtype, "int64");
|
||||
let i64_tensor = WireTensor::from_i64(vec![1], vec![-7]).unwrap();
|
||||
assert_eq!(i64_tensor.dtype, TensorDtype::I64);
|
||||
assert_eq!(
|
||||
i64_tensor.data.into_raw_view().expect("raw view").as_ref(),
|
||||
(-7_i64).to_ne_bytes().as_ref()
|
||||
);
|
||||
|
||||
let u32_tensor = WireNdArray::from_u32(vec![1], vec![42]).unwrap();
|
||||
assert_eq!(u32_tensor.dtype, "uint32");
|
||||
let u32_tensor = WireTensor::from_u32(vec![1], vec![42]).unwrap();
|
||||
assert_eq!(u32_tensor.dtype, TensorDtype::U32);
|
||||
assert_eq!(
|
||||
u32_tensor.data.into_raw_view().expect("raw view").as_ref(),
|
||||
42_u32.to_ne_bytes().as_ref()
|
||||
);
|
||||
|
||||
let bool_tensor = WireNdArray::from_bool(vec![2], vec![false, true]).unwrap();
|
||||
assert_eq!(bool_tensor.dtype, "bool");
|
||||
let bool_tensor = WireTensor::from_bool(vec![2], vec![false, true]).unwrap();
|
||||
assert_eq!(bool_tensor.dtype, TensorDtype::Bool);
|
||||
assert_eq!(
|
||||
bool_tensor.data.into_raw_view().expect("raw view"),
|
||||
vec![0, 1]
|
||||
);
|
||||
|
||||
let raw_tensor = WireNdArray::from_raw("custom", vec![3], vec![1, 2, 3]);
|
||||
assert_eq!(raw_tensor.dtype, "custom");
|
||||
assert_eq!(raw_tensor.shape, vec![3]);
|
||||
assert_eq!(
|
||||
raw_tensor.data.into_raw_view().expect("raw view"),
|
||||
vec![1, 2, 3]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn constructors_validate_shape_product() {
|
||||
let err = WireNdArray::from_f32(vec![2, 2], vec![1.0, 2.0]).unwrap_err();
|
||||
let err = WireTensor::from_f32(vec![2, 2], vec![1.0, 2.0]).unwrap_err();
|
||||
assert!(err.contains("does not match shape"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_aux_frame_shares_bytes_using_one_based_message_index() {
|
||||
let mut tensor = WireNdArray {
|
||||
dtype: "float32".to_string(),
|
||||
let mut tensor = WireTensor {
|
||||
dtype: TensorDtype::F32,
|
||||
shape: vec![2],
|
||||
data: WireArrayData::AuxIndex(1),
|
||||
};
|
||||
|
||||
@@ -22,8 +22,8 @@ use zeromq::{DealerSocket, PushSocket, SocketOptions, SubSocket, XPubSocket, Zmq
|
||||
use crate::protocol::handshake::{EngineCoreReadyResponse, HandshakeInitMessage, ReadyMessage};
|
||||
use crate::protocol::logprobs::MaybeWireLogprobs;
|
||||
use crate::protocol::multimodal::{
|
||||
MmFeatureSpec, MmField, MmFieldElem, MmFlatField, MmKwargValue, MmSlice, PlaceholderRange,
|
||||
SliceSpec,
|
||||
MmFeatureSpec, MmField, MmFieldElem, MmFlatField, MmKwargValue, MmModality, MmSlice,
|
||||
PlaceholderRange, SliceSpec,
|
||||
};
|
||||
use crate::protocol::output::{
|
||||
DpControlMessage, DpControlOutput, EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs,
|
||||
@@ -191,7 +191,7 @@ fn sample_multimodal_request() -> EngineCoreRequest {
|
||||
}),
|
||||
},
|
||||
)])),
|
||||
modality: "image".to_string(),
|
||||
modality: MmModality::Image,
|
||||
identifier: "mm-cache-key".to_string(),
|
||||
mm_position: PlaceholderRange {
|
||||
offset: 1,
|
||||
|
||||
@@ -728,7 +728,7 @@ async fn unary_generate_prepares_multimodal_input_for_engine_core() {
|
||||
assert_eq!(features.len(), 2);
|
||||
|
||||
for (feature, identifier) in features.iter().zip(["image-1", "image-2"]) {
|
||||
assert_eq!(feature.modality, "image");
|
||||
assert_eq!(feature.modality.as_str(), "image");
|
||||
assert_eq!(feature.identifier, identifier);
|
||||
assert!(feature.mm_position.length > 1);
|
||||
assert_eq!(
|
||||
|
||||
@@ -2736,7 +2736,7 @@ async fn non_stream_chat_image_url_reaches_engine_mm_features() {
|
||||
|
||||
let features = request.mm_features.as_ref().expect("multimodal features");
|
||||
assert_eq!(features.len(), 1);
|
||||
assert_eq!(features[0].modality, "image");
|
||||
assert_eq!(features[0].modality.as_str(), "image");
|
||||
assert_eq!(features[0].identifier, "image-1");
|
||||
assert!(features[0].mm_position.length > 0);
|
||||
assert!(features[0].mm_position.is_embed.is_some());
|
||||
@@ -2799,7 +2799,7 @@ async fn non_stream_chat_rejects_when_image_count_exceeds_limit_mm_per_prompt()
|
||||
default_stream_output_specs(),
|
||||
Arc::new(FakeChatBackend::with_multimodal_model_info(
|
||||
qwen_multimodal_model_info_with_limits(std::collections::HashMap::from([(
|
||||
vllm_chat::multimodal::MmLimitModality::Image,
|
||||
vllm_chat::multimodal::MmModality::Image,
|
||||
vllm_chat::multimodal::MmLimitSpec::Count(1),
|
||||
)])),
|
||||
)),
|
||||
|
||||
@@ -318,7 +318,9 @@ mod tests {
|
||||
use std::collections::{BTreeSet, HashMap};
|
||||
|
||||
use serial_test::file_serial;
|
||||
use vllm_engine_core_client::protocol::multimodal::{MmFeatureSpec, PlaceholderRange};
|
||||
use vllm_engine_core_client::protocol::multimodal::{
|
||||
MmFeatureSpec, MmModality, PlaceholderRange,
|
||||
};
|
||||
use vllm_tokenizer::test_utils::TestTokenizer;
|
||||
|
||||
use super::*;
|
||||
@@ -720,7 +722,7 @@ mod tests {
|
||||
fn lower_text_request_moves_multimodal_features_to_generate_request() {
|
||||
let features = vec![MmFeatureSpec {
|
||||
data: None,
|
||||
modality: "image".to_string(),
|
||||
modality: MmModality::Image,
|
||||
identifier: "image-1".to_string(),
|
||||
mm_position: PlaceholderRange {
|
||||
offset: 2,
|
||||
|
||||
Reference in New Issue
Block a user