[Bugfix][Rust Frontend] Fix LogprobsTensors wire schema mismatch (#53939)
Signed-off-by: jungjiyu <libraryofjiyu@gmail.com> Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Tyler Michael Smith
OpenAI Codex
parent
a27e88c862
commit
3fd72ffd7b
@@ -222,6 +222,7 @@ impl WireLogprobs {
|
|||||||
data: WireArrayData::RawView(token_ranks.into()),
|
data: WireArrayData::RawView(token_ranks.into()),
|
||||||
},
|
},
|
||||||
cu_num_generated_tokens: None,
|
cu_num_generated_tokens: None,
|
||||||
|
cu_num_generated_tokens_tensor: None,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -239,6 +240,15 @@ impl WireLogprobs {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Unlike the sibling check above, don't Debug-print the payload:
|
||||||
|
// an opaque non-None value here may embed a full tensor blob.
|
||||||
|
if self.cu_num_generated_tokens_tensor.is_some() {
|
||||||
|
bail_ext_value_decode!(
|
||||||
|
"{field_prefix}.cu_num_generated_tokens_tensor: \
|
||||||
|
expected None for per-request engine-core logprobs payload"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
let token_ids = array::decode_array2_u32(
|
let token_ids = array::decode_array2_u32(
|
||||||
self.logprob_token_ids,
|
self.logprob_token_ids,
|
||||||
&format!("{field_prefix}.logprob_token_ids"),
|
&format!("{field_prefix}.logprob_token_ids"),
|
||||||
|
|||||||
@@ -87,6 +87,7 @@ fn inline_prompt_logprobs_value() -> Value {
|
|||||||
ndarray_value("float32", &[2, 3], probs),
|
ndarray_value("float32", &[2, 3], probs),
|
||||||
ndarray_value("int64", &[2], ranks),
|
ndarray_value("int64", &[2], ranks),
|
||||||
Value::Nil,
|
Value::Nil,
|
||||||
|
Value::Nil,
|
||||||
])
|
])
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -240,6 +241,29 @@ fn decodes_inline_prompt_logprobs() {
|
|||||||
assert_eq!(logprobs, expected_prompt_logprobs());
|
assert_eq!(logprobs, expected_prompt_logprobs());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rejects_non_none_cu_num_generated_tokens_tensor() {
|
||||||
|
let Value::Array(mut fields) = inline_prompt_logprobs_value() else {
|
||||||
|
panic!("inline_prompt_logprobs_value must be an array");
|
||||||
|
};
|
||||||
|
fields[4] = Value::from(42);
|
||||||
|
|
||||||
|
let frames = vec![Bytes::from(encode_value(&output_wire_with_custom_fields(
|
||||||
|
None,
|
||||||
|
Some(Value::Array(fields)),
|
||||||
|
)))];
|
||||||
|
|
||||||
|
let error = decode_engine_core_outputs(&frames).unwrap_err();
|
||||||
|
let crate::error::Error::ExtValueDecode { message } = &error else {
|
||||||
|
panic!("expected ValueDecodeExt");
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
message,
|
||||||
|
"new_prompt_logprobs_tensors.cu_num_generated_tokens_tensor: \
|
||||||
|
expected None for per-request engine-core logprobs payload"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn decodes_big_endian_payloads() {
|
fn decodes_big_endian_payloads() {
|
||||||
let frames = vec![Bytes::from(encode_value(&output_wire_with_custom_fields(
|
let frames = vec![Bytes::from(encode_value(&output_wire_with_custom_fields(
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
|
|
||||||
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
|
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
|
||||||
|
|
||||||
|
use crate::protocol::OpaqueValue;
|
||||||
use crate::protocol::tensor::WireNdArray;
|
use crate::protocol::tensor::WireNdArray;
|
||||||
|
|
||||||
/// Python wire representation of `LogprobsLists` / `LogprobsTensors` before
|
/// Python wire representation of `LogprobsLists` / `LogprobsTensors` before
|
||||||
@@ -13,7 +14,15 @@ use crate::protocol::tensor::WireNdArray;
|
|||||||
/// into semantic per-position logprobs records.
|
/// into semantic per-position logprobs records.
|
||||||
///
|
///
|
||||||
/// Original Python definition:
|
/// Original Python definition:
|
||||||
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/outputs.py#L23-L56>
|
/// <https://github.com/vllm-project/vllm/blob/d5cadcee8641a9fcec15facb5a9157d157daa207/vllm/v1/outputs.py#L30-L106>
|
||||||
|
///
|
||||||
|
/// # Wire compatibility
|
||||||
|
///
|
||||||
|
/// Python's `LogprobsLists` / `LogprobsTensors` are `NamedTuple`s: every
|
||||||
|
/// field is serialized positionally, and `Deserialize_tuple` here hard-fails
|
||||||
|
/// on payloads longer than this struct. To preserve positional compatibility,
|
||||||
|
/// append new Python fields and add matching trailing `#[serde(default)]`
|
||||||
|
/// fields here in the same change.
|
||||||
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple)]
|
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple)]
|
||||||
pub struct WireLogprobs {
|
pub struct WireLogprobs {
|
||||||
/// Wire array with shape `[num_positions, max_num_logprobs + 1]`.
|
/// Wire array with shape `[num_positions, max_num_logprobs + 1]`.
|
||||||
@@ -31,4 +40,12 @@ pub struct WireLogprobs {
|
|||||||
/// the semantic Rust decoder rejects any other value.
|
/// the semantic Rust decoder rejects any other value.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub cu_num_generated_tokens: Option<Vec<usize>>,
|
pub cu_num_generated_tokens: Option<Vec<usize>>,
|
||||||
|
/// Device-only generation boundaries added to `LogprobsTensors` in
|
||||||
|
/// vllm-project/vllm#52242. Preserved only for wire compatibility:
|
||||||
|
/// NamedTuple encoding always emits the slot even when it is `None`,
|
||||||
|
/// so this field must exist for the 5-element payload to decode.
|
||||||
|
/// Scheduler-sliced per-request outputs should emit `None` here, and
|
||||||
|
/// the semantic Rust decoder rejects any other value.
|
||||||
|
#[serde(default)]
|
||||||
|
pub cu_num_generated_tokens_tensor: Option<OpaqueValue>,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -353,7 +353,8 @@ inline_prompt_logprobs = engine_outputs_wire(
|
|||||||
).tobytes(),
|
).tobytes(),
|
||||||
),
|
),
|
||||||
("int64", [2], np.array([3, 4], dtype=np.int64).tobytes()),
|
("int64", [2], np.array([3, 4], dtype=np.int64).tobytes()),
|
||||||
None,
|
None, # cu_num_generated_tokens
|
||||||
|
None, # cu_num_generated_tokens_tensor
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
@@ -376,7 +377,8 @@ multipart_prompt_logprobs = engine_outputs_wire(
|
|||||||
).tobytes(),
|
).tobytes(),
|
||||||
),
|
),
|
||||||
("int64", [2], np.array([3, 4], dtype=np.int64).tobytes()),
|
("int64", [2], np.array([3, 4], dtype=np.int64).tobytes()),
|
||||||
None,
|
None, # cu_num_generated_tokens
|
||||||
|
None, # cu_num_generated_tokens_tensor
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user