[Bugfix][Rust Frontend] Accept appended EngineCoreOutput fields (#56533)
Co-authored-by: Codex <noreply@openai.com> Signed-off-by: Alec Flowers <aflowers@nvidia.com>
This commit is contained in:
@@ -30,6 +30,7 @@ pub mod multimodal;
|
||||
pub mod output;
|
||||
pub mod request;
|
||||
pub mod sampling;
|
||||
mod serde_utils;
|
||||
pub mod stats;
|
||||
pub mod structured_outputs;
|
||||
pub mod tensor;
|
||||
|
||||
@@ -9,7 +9,9 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer};
|
||||
use serde_default::DefaultFromSerde;
|
||||
use serde_repr::{Deserialize_repr, Serialize_repr};
|
||||
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
|
||||
use serde_with::serde_as;
|
||||
|
||||
use super::serde_utils::AllowTrailingFields;
|
||||
use super::utility::UtilityOutput;
|
||||
use crate::error::{Error, Result, ext_value_decode};
|
||||
use crate::protocol::logprobs::MaybeWireLogprobs;
|
||||
@@ -156,11 +158,14 @@ impl EngineCoreOutput {
|
||||
///
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L186-L214>
|
||||
#[serde_as]
|
||||
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple, DefaultFromSerde)]
|
||||
struct WireEngineCoreOutputs {
|
||||
#[serde(default)]
|
||||
engine_index: u32,
|
||||
/// Outputs grouped for this client in the current engine tick.
|
||||
// Match msgspec's append-only schema evolution, including Omni extensions.
|
||||
#[serde_as(deserialize_as = "Vec<AllowTrailingFields>")]
|
||||
#[serde(default)]
|
||||
outputs: Vec<EngineCoreOutput>,
|
||||
#[serde(default)]
|
||||
@@ -404,6 +409,61 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn engine_core_outputs_ignore_appended_fields() {
|
||||
let output = EngineCoreOutput {
|
||||
request_id: "req-1".into(),
|
||||
new_token_ids: vec![42],
|
||||
finish_reason: Some(EngineCoreFinishReason::Length),
|
||||
..Default::default()
|
||||
};
|
||||
let mut fields = crate::protocol::decode_value(&encode_msgpack(&output).unwrap())
|
||||
.unwrap()
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.clone();
|
||||
// Omni appends multimodal_output, is_segment_finished, and
|
||||
// new_prompt_len_snapshot. Unknown payloads may contain nested values.
|
||||
fields.extend([
|
||||
OpaqueValue::Map(vec![(
|
||||
"audio".into(),
|
||||
OpaqueValue::Array(vec![OpaqueValue::Ext(1, vec![0, 1])]),
|
||||
)]),
|
||||
false.into(),
|
||||
OpaqueValue::Nil,
|
||||
]);
|
||||
let wire = (0, vec![OpaqueValue::Array(fields); 2]);
|
||||
let frames = [Bytes::from(encode_msgpack(&wire).unwrap())];
|
||||
let decoded = decode_engine_core_outputs(&frames).unwrap();
|
||||
// A second output also checks that the first output's tail was consumed.
|
||||
assert_eq!(decoded.as_request_batch().unwrap().outputs, vec![output; 2]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn engine_core_output_requires_valid_known_fields() {
|
||||
for fields in [
|
||||
vec![],
|
||||
vec!["req-1".into()],
|
||||
vec!["req-1".into(), OpaqueValue::Nil],
|
||||
vec!["req-1".into(), OpaqueValue::Array(vec![]), false.into()],
|
||||
] {
|
||||
let wire = (0, vec![OpaqueValue::Array(fields)]);
|
||||
let frames = [Bytes::from(encode_msgpack(&wire).unwrap())];
|
||||
assert!(decode_engine_core_outputs(&frames).is_err());
|
||||
}
|
||||
let wire = (0, vec![("req-1", vec![42_u32])]);
|
||||
let frames = [Bytes::from(encode_msgpack(&wire).unwrap())];
|
||||
let decoded = decode_engine_core_outputs(&frames).unwrap();
|
||||
assert_eq!(
|
||||
decoded.as_request_batch().unwrap().outputs,
|
||||
vec![EngineCoreOutput {
|
||||
request_id: "req-1".into(),
|
||||
new_token_ids: vec![42],
|
||||
..Default::default()
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn engine_core_outputs_classify_request_batch() {
|
||||
let outputs = WireEngineCoreOutputs {
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::fmt;
|
||||
use std::marker::PhantomData;
|
||||
|
||||
use serde::de::{IgnoredAny, SeqAccess, Visitor, value::SeqAccessDeserializer};
|
||||
use serde::{Deserialize, Deserializer};
|
||||
use serde_with::DeserializeAs;
|
||||
|
||||
/// Decode an array-like struct using its derived field types and defaults,
|
||||
/// skipping appended fields as Python's msgspec does. Opt in only for schemas
|
||||
/// whose extensions preserve the known field order and semantics.
|
||||
pub(super) struct AllowTrailingFields;
|
||||
|
||||
impl<'de, T: Deserialize<'de>> DeserializeAs<'de, T> for AllowTrailingFields {
|
||||
fn deserialize_as<D>(deserializer: D) -> Result<T, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
struct ArrayVisitor<T>(PhantomData<T>);
|
||||
|
||||
impl<'de, T: Deserialize<'de>> Visitor<'de> for ArrayVisitor<T> {
|
||||
type Value = T;
|
||||
|
||||
fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("an array-like struct")
|
||||
}
|
||||
|
||||
fn visit_seq<A>(self, mut seq: A) -> Result<T, A::Error>
|
||||
where
|
||||
A: SeqAccess<'de>,
|
||||
{
|
||||
let value = T::deserialize(SeqAccessDeserializer::new(&mut seq))?;
|
||||
while seq.next_element::<IgnoredAny>()?.is_some() {}
|
||||
Ok(value)
|
||||
}
|
||||
}
|
||||
|
||||
deserializer.deserialize_seq(ArrayVisitor(PhantomData))
|
||||
}
|
||||
}
|
||||
@@ -2641,6 +2641,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
let multi_connector_stats_hex =
|
||||
lines.next().expect("missing MultiConnector stats fixture line");
|
||||
let ready_response_hex = lines.next().expect("missing ready response fixture line");
|
||||
let extended_outputs_hex = lines.next().expect("missing extended outputs fixture line");
|
||||
|
||||
let request_bytes = hex::decode(request_hex).unwrap();
|
||||
let multimodal_request_bytes = hex::decode(multimodal_request_hex).unwrap();
|
||||
@@ -2722,6 +2723,14 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
));
|
||||
|
||||
let decoded_outputs: EngineCoreOutputs = rmp_serde::from_slice(&outputs_bytes).unwrap();
|
||||
// Match msgspec's base-schema result for the same extended Python message.
|
||||
let extended_frames = [bytes::Bytes::from(
|
||||
hex::decode(extended_outputs_hex).unwrap(),
|
||||
)];
|
||||
assert_eq!(
|
||||
decode_engine_core_outputs(&extended_frames).unwrap(),
|
||||
decoded_outputs
|
||||
);
|
||||
expect_test::expect![[r#"
|
||||
RequestBatch(
|
||||
RequestBatchOutputs {
|
||||
|
||||
@@ -101,6 +101,14 @@ class EngineCoreOutput(
|
||||
num_nans_in_logits: int = 0
|
||||
mm_cache_miss_hashes: list[str] | None = None
|
||||
new_sampling_mask: object | None = None
|
||||
spec_decode_metrics: object | None = None
|
||||
|
||||
|
||||
class ExtendedEngineCoreOutput(EngineCoreOutput):
|
||||
# Match Omni's append-only schema without requiring Omni or torch.
|
||||
multimodal_output: dict[str, object] | None = None
|
||||
is_segment_finished: bool = False
|
||||
new_prompt_len_snapshot: int | None = None
|
||||
|
||||
|
||||
class EngineCoreOutputs(
|
||||
@@ -207,6 +215,21 @@ outputs = EngineCoreOutputs(
|
||||
finished_requests={"req-1"},
|
||||
)
|
||||
|
||||
extended_outputs = EngineCoreOutputs(
|
||||
outputs=[
|
||||
ExtendedEngineCoreOutput(
|
||||
**msgspec.structs.asdict(outputs.outputs[0]),
|
||||
multimodal_output={"audio": b"\x00\x01"},
|
||||
is_segment_finished=True,
|
||||
new_prompt_len_snapshot=12,
|
||||
)
|
||||
],
|
||||
finished_requests=outputs.finished_requests,
|
||||
)
|
||||
extended_outputs_bytes = msgspec.msgpack.encode(extended_outputs)
|
||||
# The ordinary frontend's schema ignores even non-default extension values.
|
||||
assert msgspec.msgpack.decode(extended_outputs_bytes, type=EngineCoreOutputs) == outputs
|
||||
|
||||
sampling_mask_wire = [
|
||||
[
|
||||
"<i4",
|
||||
@@ -509,3 +532,4 @@ print(msgspec.msgpack.encode(nixl_stats).hex())
|
||||
print(msgspec.msgpack.encode(mooncake_stats).hex())
|
||||
print(msgspec.msgpack.encode(multi_connector_stats).hex())
|
||||
print(msgspec.msgpack.encode(ready_response).hex())
|
||||
print(extended_outputs_bytes.hex())
|
||||
|
||||
Reference in New Issue
Block a user