[Rust Frontend] Return sampling masks over gRPC (#56777)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Biswa Panda <biswa.panda@gmail.com>
This commit is contained in:
co-authored by
Bugen Zhao
parent
d5f0a6e829
commit
95d723700b
@@ -1,7 +1,7 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
mod array;
|
||||
pub(crate) mod array;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
mod wire;
|
||||
|
||||
@@ -50,7 +50,7 @@ pub(super) fn decode_array2_u32(
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn decode_array1_u32(
|
||||
pub(crate) fn decode_array1_u32(
|
||||
value: WireNdArray,
|
||||
field: &str,
|
||||
frames: &[Bytes],
|
||||
|
||||
@@ -30,6 +30,7 @@ pub mod multimodal;
|
||||
pub mod output;
|
||||
pub mod request;
|
||||
pub mod sampling;
|
||||
pub mod sampling_mask;
|
||||
mod serde_utils;
|
||||
pub mod stats;
|
||||
pub mod structured_outputs;
|
||||
|
||||
@@ -15,6 +15,7 @@ use super::serde_utils::AllowTrailingFields;
|
||||
use super::utility::UtilityOutput;
|
||||
use crate::error::{Error, Result, ext_value_decode};
|
||||
use crate::protocol::logprobs::MaybeWireLogprobs;
|
||||
use crate::protocol::sampling_mask::MaybeWireSamplingMask;
|
||||
use crate::protocol::stats::{PrefillStats, SchedulerStats};
|
||||
use crate::protocol::{OpaqueValue, decode_msgpack};
|
||||
|
||||
@@ -127,7 +128,7 @@ pub struct EngineCoreOutput {
|
||||
#[serde(default)]
|
||||
pub mm_cache_miss_hashes: Option<Vec<String>>,
|
||||
#[serde(default)]
|
||||
pub new_sampling_mask: Option<OpaqueValue>,
|
||||
pub new_sampling_mask: Option<MaybeWireSamplingMask>,
|
||||
/// Per-request speculative-decoding acceptance metrics, set on the final
|
||||
/// output when `--per-request-spec-decode-metrics` is enabled. Opaque here;
|
||||
/// the Rust frontend does not yet surface it in responses.
|
||||
@@ -150,6 +151,9 @@ impl EngineCoreOutput {
|
||||
self.new_prompt_logprobs_tensors = (self.new_prompt_logprobs_tensors.take())
|
||||
.map(|value| value.resolve(frames, "new_prompt_logprobs_tensors"))
|
||||
.transpose()?;
|
||||
self.new_sampling_mask = (self.new_sampling_mask.take())
|
||||
.map(|value| value.resolve(frames, "new_sampling_mask"))
|
||||
.transpose()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,237 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::ops::{Deref, DerefMut};
|
||||
|
||||
use bytes::Bytes;
|
||||
use enum_as_inner::EnumAsInner;
|
||||
use serde::{Deserialize, Deserializer, Serialize};
|
||||
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
|
||||
|
||||
use crate::error::{Error, Result, bail_ext_value_decode};
|
||||
use crate::protocol::logprobs::array::decode_array1_u32;
|
||||
use crate::protocol::tensor::WireNdArray;
|
||||
|
||||
/// Decoded sampling-mask support sets, one row per generated token position.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct SamplingMask {
|
||||
pub rows: Vec<Vec<u32>>,
|
||||
}
|
||||
|
||||
/// Python `SamplingMaskLists` tuple before ndarray raw views are resolved.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple)]
|
||||
pub struct WireSamplingMask {
|
||||
token_ids: WireNdArray,
|
||||
#[serde(default)]
|
||||
offsets: Option<WireNdArray>,
|
||||
#[serde(default)]
|
||||
cu_num_generated_tokens: Option<Vec<usize>>,
|
||||
}
|
||||
|
||||
/// Sampling-mask field while it transitions from Python wire data to rows.
|
||||
#[derive(Debug, Clone, PartialEq, EnumAsInner)]
|
||||
pub enum MaybeWireSamplingMask {
|
||||
Wire(Box<WireSamplingMask>),
|
||||
Direct(SamplingMask),
|
||||
}
|
||||
|
||||
impl Deref for MaybeWireSamplingMask {
|
||||
type Target = SamplingMask;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
match self {
|
||||
Self::Wire(_) => panic!("sampling mask is still in wire format"),
|
||||
Self::Direct(value) => value,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl DerefMut for MaybeWireSamplingMask {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
match self {
|
||||
Self::Wire(_) => panic!("sampling mask is still in wire format"),
|
||||
Self::Direct(value) => value,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for MaybeWireSamplingMask {
|
||||
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
WireSamplingMask::deserialize(deserializer).map(|value| Self::Wire(Box::new(value)))
|
||||
}
|
||||
}
|
||||
|
||||
impl Serialize for MaybeWireSamplingMask {
|
||||
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
match self {
|
||||
Self::Wire(value) => value.serialize(serializer),
|
||||
Self::Direct(value) => WireSamplingMask::from_direct(value)
|
||||
.map_err(serde::ser::Error::custom)?
|
||||
.serialize(serializer),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl MaybeWireSamplingMask {
|
||||
pub(super) fn resolve(self, frames: &[Bytes], field_prefix: &str) -> Result<Self> {
|
||||
match self {
|
||||
Self::Direct(value) => Ok(Self::Direct(value)),
|
||||
Self::Wire(value) => value.resolve(frames, field_prefix).map(Self::Direct),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl WireSamplingMask {
|
||||
fn from_direct(value: &SamplingMask) -> std::result::Result<Self, String> {
|
||||
let [row] = value.rows.as_slice() else {
|
||||
return Err(format!(
|
||||
"per-request engine-core sampling mask must contain exactly one row, got {}",
|
||||
value.rows.len()
|
||||
));
|
||||
};
|
||||
let token_ids = row.iter().map(|&token_id| i64::from(token_id)).collect::<Vec<_>>();
|
||||
Ok(Self {
|
||||
token_ids: WireNdArray::from_i64(vec![token_ids.len()], token_ids)?,
|
||||
offsets: None,
|
||||
cu_num_generated_tokens: None,
|
||||
})
|
||||
}
|
||||
|
||||
fn resolve(self, frames: &[Bytes], field_prefix: &str) -> Result<SamplingMask> {
|
||||
if let Some(indices) = self.cu_num_generated_tokens {
|
||||
bail_ext_value_decode!(
|
||||
"{field_prefix}.cu_num_generated_tokens: expected None for per-request engine-core sampling-mask payload, got {indices:?}"
|
||||
);
|
||||
}
|
||||
|
||||
let token_ids =
|
||||
decode_array1_u32(self.token_ids, &format!("{field_prefix}.token_ids"), frames)?;
|
||||
let Some(offsets) = self.offsets else {
|
||||
return Ok(SamplingMask {
|
||||
rows: vec![token_ids],
|
||||
});
|
||||
};
|
||||
let offsets = decode_array1_u32(offsets, &format!("{field_prefix}.offsets"), frames)?;
|
||||
if offsets.first().copied() != Some(0) {
|
||||
bail_ext_value_decode!("{field_prefix}.offsets: first offset must be zero");
|
||||
}
|
||||
if offsets.windows(2).any(|pair| pair[0] > pair[1]) {
|
||||
bail_ext_value_decode!("{field_prefix}.offsets: offsets must be nondecreasing");
|
||||
}
|
||||
if offsets.last().copied().map(|value| value as usize) != Some(token_ids.len()) {
|
||||
bail_ext_value_decode!(
|
||||
"{field_prefix}.offsets: final offset {:?} does not match token_ids length {}",
|
||||
offsets.last(),
|
||||
token_ids.len()
|
||||
);
|
||||
}
|
||||
|
||||
let rows = offsets
|
||||
.windows(2)
|
||||
.map(|pair| token_ids[pair[0] as usize..pair[1] as usize].to_vec())
|
||||
.collect();
|
||||
Ok(SamplingMask { rows })
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::protocol::dtype::{NumpyDtype, TensorDtype};
|
||||
use crate::protocol::output::{
|
||||
EngineCoreOutput, RequestBatchOutputs, decode_engine_core_outputs,
|
||||
};
|
||||
use crate::protocol::tensor::WireArrayData;
|
||||
use crate::protocol::{decode_msgpack, encode_msgpack};
|
||||
|
||||
#[test]
|
||||
fn decodes_single_position_without_offsets() {
|
||||
let wire = WireSamplingMask {
|
||||
token_ids: WireNdArray::from_i64(vec![3], vec![2, 12, 16]).unwrap(),
|
||||
offsets: None,
|
||||
cu_num_generated_tokens: None,
|
||||
};
|
||||
let encoded = encode_msgpack(&wire).unwrap();
|
||||
let decoded: MaybeWireSamplingMask = decode_msgpack(&encoded).unwrap();
|
||||
let decoded = decoded.resolve(&[], "new_sampling_mask").unwrap();
|
||||
|
||||
assert_eq!(decoded.into_direct().unwrap().rows, vec![vec![2, 12, 16]]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decodes_csr_rows_and_rejects_bad_terminal_offset() {
|
||||
let wire = WireSamplingMask {
|
||||
token_ids: WireNdArray::from_i64(vec![4], vec![2, 12, 16, 18]).unwrap(),
|
||||
offsets: Some(WireNdArray::from_i64(vec![3], vec![0, 2, 4]).unwrap()),
|
||||
cu_num_generated_tokens: None,
|
||||
};
|
||||
let decoded = MaybeWireSamplingMask::Wire(Box::new(wire))
|
||||
.resolve(&[], "new_sampling_mask")
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
decoded.into_direct().unwrap().rows,
|
||||
vec![vec![2, 12], vec![16, 18]]
|
||||
);
|
||||
|
||||
let malformed = WireSamplingMask {
|
||||
token_ids: WireNdArray::from_i64(vec![2], vec![2, 12]).unwrap(),
|
||||
offsets: Some(WireNdArray::from_i64(vec![2], vec![0, 1]).unwrap()),
|
||||
cu_num_generated_tokens: None,
|
||||
};
|
||||
let error = MaybeWireSamplingMask::Wire(Box::new(malformed))
|
||||
.resolve(&[], "new_sampling_mask")
|
||||
.unwrap_err();
|
||||
assert!(error.to_string().contains("final offset"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decodes_sampling_mask_from_multipart_aux_frames() {
|
||||
let output = EngineCoreOutput {
|
||||
request_id: "req-multipart-mask".to_string(),
|
||||
new_token_ids: vec![16, 18],
|
||||
new_sampling_mask: Some(MaybeWireSamplingMask::Wire(Box::new(WireSamplingMask {
|
||||
token_ids: WireNdArray {
|
||||
dtype: NumpyDtype::little(TensorDtype::I32),
|
||||
shape: vec![4],
|
||||
data: WireArrayData::AuxIndex(1),
|
||||
},
|
||||
offsets: Some(WireNdArray {
|
||||
dtype: NumpyDtype::little(TensorDtype::I64),
|
||||
shape: vec![3],
|
||||
data: WireArrayData::AuxIndex(2),
|
||||
}),
|
||||
cu_num_generated_tokens: None,
|
||||
}))),
|
||||
..Default::default()
|
||||
};
|
||||
let primary = encode_msgpack(&crate::protocol::output::EngineCoreOutputs::from(
|
||||
RequestBatchOutputs {
|
||||
outputs: vec![output],
|
||||
..Default::default()
|
||||
},
|
||||
))
|
||||
.unwrap();
|
||||
let token_ids =
|
||||
[2_i32, 12, 16, 18].into_iter().flat_map(i32::to_le_bytes).collect::<Vec<_>>();
|
||||
let offsets = [0_i64, 2, 4].into_iter().flat_map(i64::to_le_bytes).collect::<Vec<_>>();
|
||||
|
||||
let decoded = decode_engine_core_outputs(&[
|
||||
Bytes::from(primary),
|
||||
Bytes::from(token_ids),
|
||||
Bytes::from(offsets),
|
||||
])
|
||||
.unwrap();
|
||||
let mask = decoded.as_request_batch().unwrap().outputs[0]
|
||||
.new_sampling_mask
|
||||
.as_ref()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(mask.rows, vec![vec![2, 12], vec![16, 18]]);
|
||||
}
|
||||
}
|
||||
@@ -2712,15 +2712,15 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
decode_value(&rmp_serde::to_vec_named(&expected_multimodal_request.mm_features).unwrap());
|
||||
assert_eq!(python_mm_features, rust_mm_features);
|
||||
|
||||
let decoded_sampling_mask_outputs: EngineCoreOutputs =
|
||||
rmp_serde::from_slice(&sampling_mask_outputs_bytes).unwrap();
|
||||
let decoded_sampling_mask_outputs =
|
||||
decode_engine_core_outputs(&[bytes::Bytes::from(sampling_mask_outputs_bytes)]).unwrap();
|
||||
let sampling_mask_output =
|
||||
&decoded_sampling_mask_outputs.as_request_batch().unwrap().outputs[0];
|
||||
assert!(sampling_mask_output.mm_cache_miss_hashes.is_none());
|
||||
assert!(matches!(
|
||||
sampling_mask_output.new_sampling_mask.as_ref(),
|
||||
Some(Value::Array(fields)) if fields.len() == 3
|
||||
));
|
||||
assert_eq!(
|
||||
sampling_mask_output.new_sampling_mask.as_ref().unwrap().rows,
|
||||
vec![vec![2, 12, 16, 17, 18]]
|
||||
);
|
||||
|
||||
let decoded_outputs: EngineCoreOutputs = rmp_serde::from_slice(&outputs_bytes).unwrap();
|
||||
// Match msgspec's base-schema result for the same extended Python message.
|
||||
|
||||
@@ -234,14 +234,10 @@ sampling_mask_wire = [
|
||||
[
|
||||
"<i4",
|
||||
[5],
|
||||
msgpack.ExtType(3, np.array([2, 12, 16, 17, 18], dtype=np.int32).tobytes()),
|
||||
],
|
||||
[
|
||||
"<i8",
|
||||
[2],
|
||||
msgpack.ExtType(3, np.array([0, 5], dtype=np.int64).tobytes()),
|
||||
msgspec.msgpack.Ext(3, np.array([2, 12, 16, 17, 18], dtype=np.int32).tobytes()),
|
||||
],
|
||||
None,
|
||||
None,
|
||||
]
|
||||
outputs_with_sampling_mask = EngineCoreOutputs(
|
||||
outputs=[
|
||||
|
||||
Reference in New Issue
Block a user