[Rust Frontend] [Refactor] Extract a newtype for utility call ID (#43405)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-05-22 08:22:11 -07:00
committed by GitHub
parent f0feb15e7f
commit 91f5b92438
9 changed files with 392 additions and 237 deletions
+2 -3
View File
@@ -10,9 +10,8 @@ use crate::client::imp::{ClientInner, run_abort_loop, run_output_dispatcher_loop
use crate::coordinator::CoordinatorHandle;
use crate::error::{Error, Result};
use crate::protocol::handshake::EngineCoreReadyResponse;
use crate::protocol::{
EngineCoreRequest, EngineCoreRequestType, EngineCoreUtilityRequest, ModelDtype,
};
use crate::protocol::utility::EngineCoreUtilityRequest;
use crate::protocol::{EngineCoreRequest, EngineCoreRequestType, ModelDtype};
use crate::transport::{self, ConnectedEngine};
pub(crate) mod imp;
+12 -6
View File
@@ -16,9 +16,10 @@ use crate::client::{AbortCause, AbortRequest};
use crate::error::{client_closed, dispatcher_closed, unexpected_dispatcher_output};
use crate::metrics::record_scheduler_stats;
use crate::protocol::stats::SchedulerStats;
use crate::protocol::utility::UtilityOutput;
use crate::protocol::{
ClassifiedEngineCoreOutputs, EngineCoreOutput, EngineCoreOutputs, EngineCoreRequestType,
UtilityOutput, encode_msgpack,
encode_msgpack,
};
use crate::transport::{ConnectedEngine, EngineId};
use crate::{Error, Result, transport};
@@ -72,7 +73,7 @@ impl ClientInner {
}
/// Allocate the next utility `call_id` and register its waiting receiver.
pub fn allocate_and_register_utility_call(&self) -> Result<(i64, UtilityReceiver)> {
pub fn allocate_and_register_utility_call(&self) -> Result<(u64, UtilityReceiver)> {
let mut registry = self.utility_reg.lock();
if registry.is_closed() {
return Err(self.closed_error());
@@ -83,7 +84,7 @@ impl ClientInner {
/// Undo a batch of utility call allocations when the fan-out send fails
/// partway through. Silently ignores unknown call ids so callers can pass
/// the full set without first filtering successful sends.
pub fn unregister_utility_calls(&self, call_ids: impl IntoIterator<Item = i64>) {
pub fn unregister_utility_calls(&self, call_ids: impl IntoIterator<Item = u64>) {
self.utility_reg.lock().unregister_many(call_ids);
}
@@ -160,7 +161,12 @@ impl ClientInner {
/// Resolve one utility output to the waiting caller. Returns `true` if a
/// waiting caller existed.
pub fn resolve_utility_output(&self, output: UtilityOutput) -> bool {
match self.utility_reg.lock().resolve(&output.call_id) {
let Some(call_id) = output.call_id.as_u64() else {
// Currently, all utility call issued by the client should have unsigned call IDs.
return false;
};
match self.utility_reg.lock().resolve(&call_id) {
Some(sender) => {
sender.send(Ok(output)).unwrap_or_default();
true
@@ -342,13 +348,13 @@ pub(crate) async fn run_output_dispatcher_loop(
let call_id = utility.output.call_id;
if inner.resolve_utility_output(utility.output) {
trace!(
call_id,
%call_id,
engine_index = utility.engine_index,
"resolved utility output"
);
} else {
warn!(
call_id,
%call_id,
engine_index = utility.engine_index,
"dropping output for unexpected utility call"
);
@@ -1,5 +1,5 @@
use std::collections::BTreeMap;
use std::sync::atomic::{AtomicI64, Ordering};
use std::sync::atomic::{AtomicU64, Ordering};
use tokio::sync::{mpsc, oneshot};
use tracing::trace;
@@ -7,8 +7,9 @@ use tracing::trace;
use crate::EngineId;
use crate::client::stream::EngineCoreStreamOutput;
use crate::error::{Error, Result};
use crate::protocol::EngineCoreOutput;
use crate::protocol::stats::SchedulerStats;
use crate::protocol::{EngineCoreOutput, UtilityOutput};
use crate::protocol::utility::UtilityOutput;
use crate::transport::ConnectedEngine;
pub type OutputSender = mpsc::UnboundedSender<Result<EngineCoreStreamOutput>>;
@@ -266,15 +267,15 @@ impl RequestRegistry {
#[derive(Debug)]
pub struct UtilityRegistry {
closed: bool,
next_call_id: AtomicI64,
utility_calls: BTreeMap<i64, UtilitySender>,
next_call_id: AtomicU64,
utility_calls: BTreeMap<u64, UtilitySender>,
}
impl Default for UtilityRegistry {
fn default() -> Self {
Self {
closed: false,
next_call_id: AtomicI64::new(1),
next_call_id: AtomicU64::new(1),
utility_calls: BTreeMap::default(),
}
}
@@ -283,7 +284,7 @@ impl Default for UtilityRegistry {
impl UtilityRegistry {
/// Allocate the next utility `call_id` and register a newly added utility
/// call.
pub fn allocate_and_register(&mut self) -> (i64, UtilityReceiver) {
pub fn allocate_and_register(&mut self) -> (u64, UtilityReceiver) {
let call_id = self.next_call_id.fetch_add(1, Ordering::Relaxed);
let (tx, rx) = oneshot::channel();
self.utility_calls.insert(call_id, tx);
@@ -291,14 +292,14 @@ impl UtilityRegistry {
}
/// Resolve a utility output to its waiting receiver.
pub fn resolve(&mut self, call_id: &i64) -> Option<UtilitySender> {
pub fn resolve(&mut self, call_id: &u64) -> Option<UtilitySender> {
self.utility_calls.remove(call_id)
}
/// Drop a batch of registered utility calls without delivering a result.
/// Used to roll back allocations when the dispatch fan-out fails before
/// every engine could accept the request.
pub fn unregister_many(&mut self, call_ids: impl IntoIterator<Item = i64>) {
pub fn unregister_many(&mut self, call_ids: impl IntoIterator<Item = u64>) {
for call_id in call_ids {
self.utility_calls.remove(&call_id);
}
@@ -315,7 +316,7 @@ impl UtilityRegistry {
}
#[cfg(test)]
pub fn contains(&self, call_id: i64) -> bool {
pub fn contains(&self, call_id: u64) -> bool {
self.utility_calls.contains_key(&call_id)
}
+5 -3
View File
@@ -4,6 +4,8 @@ use std::time::Duration;
use thiserror::Error;
use thiserror_ext::Macro;
use crate::protocol::utility::UtilityCallId;
pub type Result<T> = std::result::Result<T, Error>;
/// Public error type for the Rust engine-core client.
@@ -70,17 +72,17 @@ pub enum Error {
#[error("utility call `{method}` failed (call_id={call_id}): {message}")]
UtilityCallFailed {
method: String,
call_id: i64,
call_id: UtilityCallId,
message: String,
},
#[error("utility call `{method}` returned an invalid result (call_id={call_id}): {message}")]
UtilityResultDecode {
method: String,
call_id: i64,
call_id: UtilityCallId,
message: String,
},
#[error("utility call `{method}` closed unexpectedly (call_id={call_id})")]
UtilityCallClosed { method: String, call_id: i64 },
UtilityCallClosed { method: String, call_id: u64 },
/// A special variant to allow cloning the same error.
#[error(transparent)]
@@ -2,7 +2,8 @@ use std::collections::BTreeSet;
use enum_as_inner::EnumAsInner;
use super::{EngineCoreOutput, EngineCoreOutputs, UtilityOutput};
use super::utility::UtilityOutput;
use super::{EngineCoreOutput, EngineCoreOutputs};
use crate::protocol::stats::SchedulerStats;
/// Data-parallel control notifications multiplexed through `EngineCoreOutputs`.
@@ -150,7 +151,7 @@ mod tests {
fn engine_core_outputs_classify_utility() {
let outputs = EngineCoreOutputs {
utility_output: Some(UtilityOutput {
call_id: 42,
call_id: 42_u64.into(),
failure_message: None,
result: None,
}),
@@ -201,7 +202,7 @@ mod tests {
..Default::default()
}],
utility_output: Some(UtilityOutput {
call_id: 1,
call_id: 1_u64.into(),
failure_message: None,
result: None,
}),
+2 -187
View File
@@ -14,6 +14,7 @@ use crate::error::{Error, Result};
use crate::protocol::logprobs::MaybeWireLogprobs;
use crate::protocol::multimodal::MmFeatures;
use crate::protocol::stats::{PrefillStats, SchedulerStats};
use crate::protocol::utility::UtilityOutput;
// TODO: This module currently mixes reusable frontend-facing semantic types
// (for example `FinishReason`, `StopReason`, `RequestOutputKind`, and future
@@ -42,6 +43,7 @@ pub mod logprobs;
pub mod multimodal;
pub mod stats;
pub mod tensor;
pub mod utility;
pub use classified_outputs::{
ClassifiedEngineCoreOutputs, DpControlMessage, RequestBatchOutputs, UtilityCallOutput,
};
@@ -373,51 +375,6 @@ impl EngineCoreRequest {
}
}
/// Engine-core utility call payload sent from frontend to engine.
///
/// Original Python payload shape:
/// `(client_index, call_id, method_name, args)`
#[derive(Debug, Clone, PartialEq, Serialize_tuple)]
pub struct EngineCoreUtilityRequest {
pub client_index: u32,
pub call_id: i64,
pub method_name: String,
pub args: OpaqueValue,
}
impl EngineCoreUtilityRequest {
/// Create a new utility request with the given strongly typed arguments,
/// encoding them into the expected msgpack value format.
pub fn new<T>(
client_index: u32,
call_id: i64,
method_name: impl Into<String>,
args: T,
) -> Result<Self>
where
T: Serialize + std::fmt::Debug,
{
let args = rmpv::ext::to_value(&args).map_err(|error| Error::Encode {
target_type: type_name::<T>(),
message: format!(
"failed to encode utility args `{args:?}`: {}",
error.to_report_string()
),
})?;
let args = match args {
Value::Nil => Value::Array(Vec::new()),
other => other,
};
Ok(Self {
client_index,
call_id,
method_name: method_name.into(),
args,
})
}
}
/// Engine-core output for a single request.
///
/// Original Python definition:
@@ -464,70 +421,6 @@ impl EngineCoreOutput {
}
}
/// Result of a utility call.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L174-L183>
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple, DefaultFromSerde)]
pub struct UtilityOutput {
pub call_id: i64,
/// Non-`None` implies the call failed and `result` should be ignored.
#[serde(default)]
pub failure_message: Option<String>,
#[serde(default)]
pub result: Option<UtilityResultEnvelope>,
}
/// Python `UtilityResult` wrapper carried inside `UtilityOutput.result`.
///
/// Upstream reference:
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/serial_utils.py#L178-L185>
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple)]
pub struct UtilityResultEnvelope {
/// Recursive type information encoded on Python side, serving as the hint
/// for deserialization. We don't care it here as in Rust frontend all
/// utility calls are strongly-typed.
#[serde(default)]
type_info: Option<OpaqueValue>,
/// The actual utility result.
#[serde(default = "default_opaque_value_nil")]
result: OpaqueValue,
}
impl UtilityResultEnvelope {
/// Create a utility result envelope without type information.
pub fn without_type_info(result: OpaqueValue) -> Self {
Self {
type_info: None,
result,
}
}
}
impl UtilityOutput {
/// Decode the typed result of a utility call.
pub fn into_typed_result<T>(self, method: &str) -> Result<T>
where
T: serde::de::DeserializeOwned,
{
if let Some(message) = self.failure_message {
return Err(Error::UtilityCallFailed {
method: method.to_string(),
call_id: self.call_id,
message,
});
}
let result = self.result.map(|e| e.result).unwrap_or(Value::Nil);
rmpv::ext::from_value(result).map_err(|error| Error::UtilityResultDecode {
method: method.to_string(),
call_id: self.call_id,
message: error.to_report_string(),
})
}
}
/// Batch of engine-core outputs returned to a frontend client.
///
/// Original Python definition:
@@ -601,13 +494,6 @@ mod tests {
use super::*;
fn utility_result_value<T>(value: T) -> UtilityResultEnvelope
where
T: Serialize,
{
UtilityResultEnvelope::without_type_info(rmpv::ext::to_value(value).unwrap())
}
#[test]
fn engine_core_request_serializes_as_full_array() {
let request = EngineCoreRequest {
@@ -673,77 +559,6 @@ mod tests {
);
}
#[test]
fn utility_request_serializes_as_tuple_payload() {
let request = EngineCoreUtilityRequest::new(7, 42, "is_sleeping", ()).unwrap();
let encoded = encode_msgpack(&request).unwrap();
let value = decode_value(&encoded).unwrap();
let array = match value {
Value::Array(array) => array,
other => panic!("expected utility request array, got {other:?}"),
};
assert_eq!(array.len(), 4);
assert_eq!(array[0], Value::from(7));
assert_eq!(array[1], Value::from(42));
assert_eq!(array[2], Value::from("is_sleeping"));
assert_eq!(array[3], Value::Array(Vec::new()));
}
#[test]
fn utility_output_decodes_typed_result() {
let output = UtilityOutput {
call_id: 9,
failure_message: None,
result: Some(utility_result_value(true)),
};
assert!(output.into_typed_result::<bool>("is_sleeping").unwrap());
}
#[test]
fn utility_output_reports_failure_message() {
let error = UtilityOutput {
call_id: 9,
failure_message: Some("boom".to_string()),
result: None,
}
.into_typed_result::<bool>("is_sleeping")
.unwrap_err();
assert!(matches!(
error,
Error::UtilityCallFailed {
method,
call_id,
message
} if method == "is_sleeping" && call_id == 9 && message == "boom"
));
}
#[test]
fn utility_output_decodes_missing_result_as_unit() {
UtilityOutput {
call_id: 3,
failure_message: None,
result: None,
}
.into_typed_result::<()>("reset_mm_cache")
.unwrap();
}
#[test]
fn utility_output_decodes_nil_result_as_unit() {
UtilityOutput {
call_id: 4,
failure_message: None,
result: Some(UtilityResultEnvelope::without_type_info(Value::Nil)),
}
.into_typed_result::<()>("sleep")
.unwrap();
}
#[test]
fn decode_msgpack_includes_type_name_and_value_fallback() {
let error = decode_msgpack::<u64>(
@@ -0,0 +1,330 @@
use std::any::type_name;
use std::fmt;
use rmpv::Value;
use serde::{Deserialize, Serialize};
use serde_default::DefaultFromSerde;
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
use thiserror_ext::AsReport;
use super::{OpaqueValue, default_opaque_value_nil};
use crate::error::{Error, Result};
/// Utility call id as carried on the engine-core MessagePack wire.
///
/// Python emits utility ids as MessagePack integers, including values that may
/// require unsigned 64-bit encoding. Keep MessagePack's signed/unsigned
/// integer distinction instead of flattening to `i64` or `u64` at decode time.
#[derive(Clone, Copy, PartialEq)]
pub struct UtilityCallId(rmpv::Integer);
impl UtilityCallId {
/// Returns the integer represented as `u64` if possible, or else `None`.
/// This is the typical case for utility calls.
pub fn as_u64(self) -> Option<u64> {
self.0.as_u64()
}
/// Returns the integer represented as `i64` if possible, or else `None`.
pub fn as_i64(self) -> Option<i64> {
self.0.as_i64()
}
}
impl Default for UtilityCallId {
fn default() -> Self {
Self(0_u64.into())
}
}
impl From<u64> for UtilityCallId {
fn from(value: u64) -> Self {
Self(value.into())
}
}
impl TryFrom<Value> for UtilityCallId {
type Error = String;
fn try_from(value: Value) -> std::result::Result<Self, Self::Error> {
match value {
Value::Integer(value) => Ok(UtilityCallId(value)),
other => Err(format!(
"expected a MessagePack integer utility call id, got {other}"
)),
}
}
}
impl PartialEq<u64> for UtilityCallId {
fn eq(&self, other: &u64) -> bool {
self.as_u64() == Some(*other)
}
}
impl fmt::Debug for UtilityCallId {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Display::fmt(&self.0, f)
}
}
impl fmt::Display for UtilityCallId {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Display::fmt(&self.0, f)
}
}
impl Serialize for UtilityCallId {
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
Value::Integer(self.0).serialize(serializer)
}
}
impl<'de> Deserialize<'de> for UtilityCallId {
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
let value = Value::deserialize(deserializer)?;
match value {
Value::Integer(value) => Ok(UtilityCallId(value)),
other => Err(serde::de::Error::custom(format!(
"expected a MessagePack integer utility call id, got {other}"
))),
}
}
}
/// Engine-core utility call payload sent from frontend to engine.
///
/// Original Python payload shape:
/// `(client_index, call_id, method_name, args)`
#[derive(Debug, Clone, PartialEq, Serialize_tuple)]
pub struct EngineCoreUtilityRequest {
pub client_index: u32,
pub call_id: UtilityCallId,
pub method_name: String,
pub args: OpaqueValue,
}
impl EngineCoreUtilityRequest {
/// Create a new utility request with the given strongly typed arguments,
/// encoding them into the expected msgpack value format.
pub fn new<T>(
client_index: u32,
call_id: u64,
method_name: impl Into<String>,
args: T,
) -> Result<Self>
where
T: Serialize + std::fmt::Debug,
{
let args = rmpv::ext::to_value(&args).map_err(|error| Error::Encode {
target_type: type_name::<T>(),
message: format!(
"failed to encode utility args `{args:?}`: {}",
error.to_report_string()
),
})?;
let args = match args {
Value::Nil => Value::Array(Vec::new()),
other => other,
};
Ok(Self {
client_index,
call_id: UtilityCallId::from(call_id),
method_name: method_name.into(),
args,
})
}
}
/// Result of a utility call.
///
/// Original Python definition:
/// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/__init__.py#L174-L183>
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple, DefaultFromSerde)]
pub struct UtilityOutput {
pub call_id: UtilityCallId,
/// Non-`None` implies the call failed and `result` should be ignored.
#[serde(default)]
pub failure_message: Option<String>,
#[serde(default)]
pub result: Option<UtilityResultEnvelope>,
}
/// Python `UtilityResult` wrapper carried inside `UtilityOutput.result`.
///
/// Upstream reference:
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/serial_utils.py#L178-L185>
#[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple)]
pub struct UtilityResultEnvelope {
/// Recursive type information encoded on Python side, serving as the hint
/// for deserialization. We don't care it here as in Rust frontend all
/// utility calls are strongly-typed.
#[serde(default)]
type_info: Option<OpaqueValue>,
/// The actual utility result.
#[serde(default = "default_opaque_value_nil")]
result: OpaqueValue,
}
impl UtilityResultEnvelope {
/// Create a utility result envelope without type information.
pub fn without_type_info(result: OpaqueValue) -> Self {
Self {
type_info: None,
result,
}
}
}
impl UtilityOutput {
/// Decode the typed result of a utility call.
pub fn into_typed_result<T>(self, method: &str) -> Result<T>
where
T: serde::de::DeserializeOwned,
{
if let Some(message) = self.failure_message {
return Err(Error::UtilityCallFailed {
method: method.to_string(),
call_id: self.call_id,
message,
});
}
let result = self.result.map(|e| e.result).unwrap_or(Value::Nil);
rmpv::ext::from_value(result).map_err(|error| Error::UtilityResultDecode {
method: method.to_string(),
call_id: self.call_id,
message: error.to_report_string(),
})
}
}
#[cfg(test)]
mod tests {
use rmpv::Value;
use serde::Serialize;
use super::{EngineCoreUtilityRequest, UtilityOutput, UtilityResultEnvelope};
use crate::Error;
use crate::protocol::{decode_msgpack, decode_value, encode_msgpack};
fn utility_result_value<T>(value: T) -> UtilityResultEnvelope
where
T: Serialize,
{
UtilityResultEnvelope::without_type_info(rmpv::ext::to_value(value).unwrap())
}
#[test]
fn utility_request_serializes_as_tuple_payload() {
let request = EngineCoreUtilityRequest::new(7, 42, "is_sleeping", ()).unwrap();
let encoded = encode_msgpack(&request).unwrap();
let value = decode_value(&encoded).unwrap();
let array = match value {
Value::Array(array) => array,
other => panic!("expected utility request array, got {other:?}"),
};
assert_eq!(array.len(), 4);
assert_eq!(array[0], Value::from(7));
assert_eq!(array[1], Value::from(42));
assert_eq!(array[2], Value::from("is_sleeping"));
assert_eq!(array[3], Value::Array(Vec::new()));
}
#[test]
fn utility_output_decodes_typed_result() {
let output = UtilityOutput {
call_id: 9_u64.into(),
failure_message: None,
result: Some(utility_result_value(true)),
};
assert!(output.into_typed_result::<bool>("is_sleeping").unwrap());
}
#[test]
fn utility_output_decodes_unsigned_64_bit_call_id() {
let value = Value::Array(vec![Value::from(u64::MAX), Value::Nil, Value::Nil]);
let mut encoded = Vec::new();
rmpv::encode::write_value(&mut encoded, &value).unwrap();
let output: UtilityOutput = decode_msgpack(&encoded).unwrap();
assert_eq!(output.call_id.as_u64(), Some(u64::MAX));
}
#[test]
fn utility_output_decodes_signed_negative_call_id() {
let value = Value::Array(vec![Value::from(-1), Value::Nil, Value::Nil]);
let mut encoded = Vec::new();
rmpv::encode::write_value(&mut encoded, &value).unwrap();
let output: UtilityOutput = decode_msgpack(&encoded).unwrap();
assert_eq!(output.call_id.as_u64(), None);
}
#[test]
fn utility_output_decodes_other_negative_call_id() {
let value = Value::Array(vec![Value::from(-2), Value::Nil, Value::Nil]);
let mut encoded = Vec::new();
rmpv::encode::write_value(&mut encoded, &value).unwrap();
let output: UtilityOutput = decode_msgpack(&encoded).unwrap();
assert_eq!(output.call_id.as_u64(), None);
assert_eq!(output.call_id.as_i64(), Some(-2));
}
#[test]
fn utility_output_reports_failure_message() {
let error = UtilityOutput {
call_id: 9_u64.into(),
failure_message: Some("boom".to_string()),
result: None,
}
.into_typed_result::<bool>("is_sleeping")
.unwrap_err();
assert!(matches!(
error,
Error::UtilityCallFailed {
method,
call_id,
message
} if method == "is_sleeping" && call_id == 9 && message == "boom"
));
}
#[test]
fn utility_output_decodes_missing_result_as_unit() {
UtilityOutput {
call_id: 3_u64.into(),
failure_message: None,
result: None,
}
.into_typed_result::<()>("reset_mm_cache")
.unwrap();
}
#[test]
fn utility_output_decodes_nil_result_as_unit() {
UtilityOutput {
call_id: 4_u64.into(),
failure_message: None,
result: Some(UtilityResultEnvelope::without_type_info(Value::Nil)),
}
.into_typed_result::<()>("sleep")
.unwrap();
}
}
+16 -16
View File
@@ -24,10 +24,10 @@ use crate::protocol::multimodal::{
};
use crate::protocol::stats::SchedulerStats;
use crate::protocol::tensor::WireTensor;
use crate::protocol::utility::{UtilityOutput, UtilityResultEnvelope};
use crate::protocol::{
EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, EngineCoreRequest,
EngineCoreRequestType, EngineCoreSamplingParams, UtilityOutput, UtilityResultEnvelope,
decode_engine_core_outputs,
EngineCoreRequestType, EngineCoreSamplingParams, decode_engine_core_outputs,
};
use crate::test_utils::{
IpcNamespace, setup_bootstrapped_mock_engine, setup_mock_engine_connections,
@@ -878,7 +878,7 @@ async fn client_fail_closes_when_main_output_path_receives_dp_control() {
push,
EngineCoreOutputs {
utility_output: Some(UtilityOutput {
call_id: 1,
call_id: 1_u64.into(),
failure_message: None,
result: None,
}),
@@ -978,7 +978,7 @@ async fn client_fail_closes_when_main_output_path_receives_mixed_shape_output()
push,
EngineCoreOutputs {
utility_output: Some(UtilityOutput {
call_id: 1,
call_id: 1_u64.into(),
failure_message: None,
result: None,
}),
@@ -1309,7 +1309,7 @@ async fn is_sleeping_wrapper_sends_typed_request_and_returns_typed_response() {
};
assert_eq!(array.len(), 4);
assert_eq!(array[0], Value::from(5));
let call_id = array[1].as_i64().expect("call_id");
let call_id = array[1].as_u64().expect("call_id");
assert_eq!(array[2], Value::from("is_sleeping"));
assert_eq!(array[3], Value::Array(Vec::new()));
@@ -1317,7 +1317,7 @@ async fn is_sleeping_wrapper_sends_typed_request_and_returns_typed_response() {
push,
EngineCoreOutputs {
utility_output: Some(UtilityOutput {
call_id,
call_id: call_id.into(),
failure_message: None,
result: Some(utility_result_value(true)),
}),
@@ -1366,13 +1366,13 @@ async fn call_utility_failure_message_surfaces_as_error() {
assert_eq!(utility[0].as_ref(), &[0x03]);
let payload = decode_value(&utility[1]);
let call_id =
payload.as_array().and_then(|array| array[1].as_i64()).expect("call_id");
payload.as_array().and_then(|array| array[1].as_u64()).expect("call_id");
send_outputs(
push,
EngineCoreOutputs {
utility_output: Some(UtilityOutput {
call_id,
call_id: call_id.into(),
failure_message: Some("boom".to_string()),
result: None,
}),
@@ -1865,13 +1865,13 @@ async fn multi_engine_abort_is_grouped_and_utility_fans_out_to_all_engines() {
Value::Array(array) => array,
other => panic!("expected utility payload array, got {other:?}"),
};
let call_id = array[1].as_i64().expect("call_id");
let call_id = array[1].as_u64().expect("call_id");
assert_eq!(array[2], Value::from("is_sleeping"));
send_outputs(
push,
EngineCoreOutputs {
utility_output: Some(UtilityOutput {
call_id,
call_id: call_id.into(),
failure_message: None,
result: Some(utility_result_value(true)),
}),
@@ -1919,13 +1919,13 @@ async fn multi_engine_abort_is_grouped_and_utility_fans_out_to_all_engines() {
Value::Array(array) => array,
other => panic!("expected utility payload array, got {other:?}"),
};
let call_id = array[1].as_i64().expect("call_id");
let call_id = array[1].as_u64().expect("call_id");
assert_eq!(array[2], Value::from("is_sleeping"));
send_outputs(
push,
EngineCoreOutputs {
utility_output: Some(UtilityOutput {
call_id,
call_id: call_id.into(),
failure_message: None,
result: Some(utility_result_value(true)),
}),
@@ -2029,14 +2029,14 @@ async fn collective_rpc_flattens_results_from_all_engines() {
Value::Array(array) => array,
other => panic!("expected utility payload array, got {other:?}"),
};
let call_id = array[1].as_i64().expect("call_id");
let call_id = array[1].as_u64().expect("call_id");
assert_eq!(array[2], Value::from("collective_rpc"));
send_outputs(
push,
EngineCoreOutputs {
utility_output: Some(UtilityOutput {
call_id,
call_id: call_id.into(),
failure_message: None,
result: Some(utility_result_value(vec!["engine-0-worker"])),
}),
@@ -2060,14 +2060,14 @@ async fn collective_rpc_flattens_results_from_all_engines() {
Value::Array(array) => array,
other => panic!("expected utility payload array, got {other:?}"),
};
let call_id = array[1].as_i64().expect("call_id");
let call_id = array[1].as_u64().expect("call_id");
assert_eq!(array[2], Value::from("collective_rpc"));
send_outputs(
push,
EngineCoreOutputs {
utility_output: Some(UtilityOutput {
call_id,
call_id: call_id.into(),
failure_message: None,
result: Some(utility_result_value(vec!["engine-1-worker"])),
}),
+11 -10
View File
@@ -27,9 +27,10 @@ use vllm_chat::{
use vllm_engine_core_client::protocol::logprobs::{
Logprobs, MaybeWireLogprobs, PositionLogprobs, TokenLogprob,
};
use vllm_engine_core_client::protocol::utility::{UtilityOutput, UtilityResultEnvelope};
use vllm_engine_core_client::protocol::{
EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, EngineCoreRequest, StopReason,
UtilityOutput, UtilityResultEnvelope, decode_value,
decode_value,
};
use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
use vllm_engine_core_client::{
@@ -369,10 +370,10 @@ fn utility_none_result() -> UtilityResultEnvelope {
UtilityResultEnvelope::without_type_info(Value::Nil)
}
fn utility_outputs(call_id: i64, result: UtilityResultEnvelope) -> EngineCoreOutputs {
fn utility_outputs(call_id: u64, result: UtilityResultEnvelope) -> EngineCoreOutputs {
EngineCoreOutputs {
utility_output: Some(UtilityOutput {
call_id,
call_id: call_id.into(),
failure_message: None,
result: Some(result),
}),
@@ -2989,7 +2990,7 @@ async fn reset_prefix_cache_route_sends_expected_utility_call() {
let payload = decode_value(&utility[1]).expect("decode utility payload");
let array = payload.as_array().expect("utility payload array");
let call_id = array[1].as_i64().expect("call id");
let call_id = array[1].as_u64().expect("call id");
assert_eq!(array[2], Value::from("reset_prefix_cache"));
assert_eq!(
@@ -3031,7 +3032,7 @@ async fn reset_mm_cache_route_sends_expected_utility_call() {
let payload = decode_value(&utility[1]).expect("decode utility payload");
let array = payload.as_array().expect("utility payload array");
let call_id = array[1].as_i64().expect("call id");
let call_id = array[1].as_u64().expect("call id");
assert_eq!(array[2], Value::from("reset_mm_cache"));
assert_eq!(array[3], Value::Array(Vec::new()));
@@ -3070,7 +3071,7 @@ async fn reset_encoder_cache_route_sends_expected_utility_call() {
let payload = decode_value(&utility[1]).expect("decode utility payload");
let array = payload.as_array().expect("utility payload array");
let call_id = array[1].as_i64().expect("call id");
let call_id = array[1].as_u64().expect("call id");
assert_eq!(array[2], Value::from("reset_encoder_cache"));
assert_eq!(array[3], Value::Array(Vec::new()));
@@ -3109,7 +3110,7 @@ async fn collective_rpc_route_sends_expected_utility_call_and_returns_results()
let payload = decode_value(&utility[1]).expect("decode utility payload");
let array = payload.as_array().expect("utility payload array");
let call_id = array[1].as_i64().expect("call id");
let call_id = array[1].as_u64().expect("call id");
assert_eq!(array[2], Value::from("collective_rpc"));
assert_eq!(
@@ -3194,7 +3195,7 @@ async fn sleep_route_uses_python_compatible_default_query_values() {
let payload = decode_value(&utility[1]).expect("decode utility payload");
let array = payload.as_array().expect("utility payload array");
let call_id = array[1].as_i64().expect("call id");
let call_id = array[1].as_u64().expect("call id");
assert_eq!(array[2], Value::from("sleep"));
assert_eq!(
@@ -3236,7 +3237,7 @@ async fn wake_up_route_without_tags_sends_none() {
let payload = decode_value(&utility[1]).expect("decode utility payload");
let array = payload.as_array().expect("utility payload array");
let call_id = array[1].as_i64().expect("call id");
let call_id = array[1].as_u64().expect("call id");
assert_eq!(array[2], Value::from("wake_up"));
assert_eq!(array[3], Value::Array(vec![Value::Nil]));
@@ -3275,7 +3276,7 @@ async fn is_sleeping_route_returns_json_payload() {
let payload = decode_value(&utility[1]).expect("decode utility payload");
let array = payload.as_array().expect("utility payload array");
let call_id = array[1].as_i64().expect("call id");
let call_id = array[1].as_u64().expect("call id");
assert_eq!(array[2], Value::from("is_sleeping"));
assert_eq!(array[3], Value::Array(Vec::new()));