[Rust Frontend] Send multimodal tensors in auxiliary frames (#49341)

Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: reidliu41 <reid201711@gmail.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Reid
2026-07-29 12:21:09 +00:00
committed by GitHub
co-authored by Bugen Zhao
parent 625871b52c
commit 242c591d5a
11 changed files with 530 additions and 159 deletions
+1 -1
View File
@@ -519,7 +519,7 @@ impl EngineCoreClient {
"registered request to engine"
);
self.inner.send_to_engine(&engine_id, EngineCoreRequestType::Add, &req).await?;
self.inner.send_request_to_engine(&engine_id, req).await?;
Ok(())
}
.await;
+40 -4
View File
@@ -6,6 +6,7 @@ use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use arc_swap::ArcSwapOption;
use bytes::Bytes;
use parking_lot::Mutex;
use thiserror_ext::AsReport as _;
use tokio::runtime::Handle;
@@ -21,17 +22,30 @@ use crate::error::{client_closed, dispatcher_closed, unexpected_dispatcher_outpu
use crate::metrics::{LoraInfoExporter, SchedulerStatsRecorder};
use crate::protocol::encode_msgpack;
use crate::protocol::output::{EngineCoreOutput, EngineCoreOutputs};
use crate::protocol::request::EngineCoreRequestType;
use crate::protocol::request::{EngineCoreRequest, EngineCoreRequestType};
use crate::protocol::stats::SchedulerStats;
use crate::protocol::utility::UtilityOutput;
use crate::transport::{ConnectedEngine, EngineId};
use crate::{Error, Result, transport};
const MSGPACK_ZERO_COPY_THRESHOLD_ENV: &str = "VLLM_MSGPACK_ZERO_COPY_THRESHOLD";
const DEFAULT_MSGPACK_ZERO_COPY_THRESHOLD: usize = 256;
fn msgpack_zero_copy_threshold() -> usize {
std::env::var(MSGPACK_ZERO_COPY_THRESHOLD_ENV)
.ok()
.and_then(|value| value.parse().ok())
.unwrap_or(DEFAULT_MSGPACK_ZERO_COPY_THRESHOLD)
}
pub(crate) struct ClientInner {
input_send: RouterSendHalf,
/// The runtime handle used for sending messages to the engine.
handle: Handle,
model_name: String,
/// Per-tensor byte threshold loaded from env variable
/// `VLLM_MSGPACK_ZERO_COPY_THRESHOLD` when this inner client is created.
msgpack_zero_copy_threshold: usize,
scheduler_stats_recorder: SchedulerStatsRecorder,
request_reg: Mutex<RequestRegistry>,
utility_reg: Mutex<UtilityRegistry>,
@@ -54,6 +68,7 @@ impl ClientInner {
input_send,
handle,
model_name,
msgpack_zero_copy_threshold: msgpack_zero_copy_threshold(),
scheduler_stats_recorder,
request_reg: Mutex::new(RequestRegistry::new(engines)),
utility_reg: Mutex::new(UtilityRegistry::default()),
@@ -229,9 +244,29 @@ impl ClientInner {
where
T: serde::Serialize + std::fmt::Debug,
{
// TODO: for `EngineCoreRequest`, split outbound tensor raw views into aux
// frames instead of always producing a single msgpack frame.
let payload = encode_msgpack(payload)?;
let payload = Bytes::from(encode_msgpack(payload)?);
self.send_encoded_to_engine(engine_id, request_type, payload, Vec::new()).await
}
/// Send an add request, moving large tensor buffers into auxiliary frames.
pub async fn send_request_to_engine(
&self,
engine_id: &EngineId,
mut payload: EngineCoreRequest,
) -> Result<()> {
let aux_frames = payload.extract_aux_frames(self.msgpack_zero_copy_threshold);
let payload = Bytes::from(encode_msgpack(&payload)?);
self.send_encoded_to_engine(engine_id, EngineCoreRequestType::Add, payload, aux_frames)
.await
}
async fn send_encoded_to_engine(
&self,
engine_id: &EngineId,
request_type: EngineCoreRequestType,
payload: Bytes,
aux_frames: Vec<Bytes>,
) -> Result<()> {
let mut input_send = self.input_send.clone();
let engine_id = engine_id.clone();
@@ -242,6 +277,7 @@ impl ClientInner {
&engine_id,
request_type.to_frame(),
payload,
aux_frames,
)
.await
})
@@ -209,17 +209,17 @@ impl WireLogprobs {
logprob_token_ids: WireNdArray {
dtype: "<i8".to_string(),
shape: vec![rows, cols],
data: WireArrayData::RawView(token_ids),
data: WireArrayData::RawView(token_ids.into()),
},
logprobs: WireNdArray {
dtype: "<f4".to_string(),
shape: vec![rows, cols],
data: WireArrayData::RawView(logprobs),
data: WireArrayData::RawView(logprobs.into()),
},
token_ranks: WireNdArray {
dtype: "<i8".to_string(),
shape: vec![rows],
data: WireArrayData::RawView(token_ranks),
data: WireArrayData::RawView(token_ranks.into()),
},
cu_num_generated_tokens: None,
})
@@ -4,6 +4,7 @@
use std::io::Cursor;
use byteorder::{BigEndian, LittleEndian, NativeEndian, ReadBytesExt};
use bytes::Bytes;
use itertools::Itertools as _;
use crate::error::{Error, Result, ext_value_decode};
@@ -126,7 +127,7 @@ pub(super) fn decode_array_metadata<Frame>(
field: &str,
frames: &[Frame],
expected_scalars: &[ScalarType],
) -> Result<(Vec<usize>, Vec<u8>, ScalarType, Endianness)>
) -> Result<(Vec<usize>, Bytes, ScalarType, Endianness)>
where
Frame: AsRef<[u8]>,
{
@@ -171,7 +172,7 @@ pub(super) fn resolve_array_bytes<Frame>(
value: WireArrayData,
field: &str,
frames: &[Frame],
) -> Result<Vec<u8>>
) -> Result<Bytes>
where
Frame: AsRef<[u8]>,
{
@@ -187,7 +188,7 @@ where
),
)
})?;
Ok(frame.as_ref().to_vec())
Ok(Bytes::copy_from_slice(frame.as_ref()))
}
}
}
@@ -3,6 +3,7 @@
use std::collections::BTreeMap;
use bytes::Bytes;
use serde::{Deserialize, Serialize};
use serde_tuple::{Deserialize_tuple, Serialize_tuple};
@@ -105,6 +106,44 @@ pub enum MmKwargValue {
List(Vec<MmKwargValue>),
}
impl MmFeatureSpec {
/// Extract large tensor buffers from this feature in serialized field order.
pub(crate) fn extract_aux_frames(&mut self, aux_frames: &mut Vec<Bytes>, threshold: usize) {
if let Some(data) = &mut self.data {
for elem in data.values_mut() {
elem.extract_aux_frames(aux_frames, threshold);
}
}
if let Some(is_embed) = &mut self.mm_position.is_embed {
is_embed.extract_aux_frame(aux_frames, threshold);
}
}
}
impl MmFieldElem {
/// Extract large tensor buffers from this field element.
fn extract_aux_frames(&mut self, aux_frames: &mut Vec<Bytes>, threshold: usize) {
if let Some(data) = &mut self.data {
data.extract_aux_frames(aux_frames, threshold);
}
}
}
impl MmKwargValue {
/// Recursively extract large tensor buffers from this nested value.
fn extract_aux_frames(&mut self, aux_frames: &mut Vec<Bytes>, threshold: usize) {
match self {
Self::Tensor(tensor) => tensor.extract_aux_frame(aux_frames, threshold),
Self::List(values) => {
for value in values {
value.extract_aux_frames(aux_frames, threshold);
}
}
Self::Int(_) | Self::Float(_) => {}
}
}
}
/// Defines how to interpret tensor data belonging to a keyword argument for
/// `MultiModalKwargsItems`, and vice versa.
///
@@ -137,6 +137,17 @@ impl EngineCoreRequest {
}
Ok(())
}
/// Extract large request tensors into ordered auxiliary frames.
pub(crate) fn extract_aux_frames(&mut self, threshold: usize) -> Vec<Bytes> {
let mut aux_frames = Vec::new();
if let Some(features) = &mut self.mm_features {
for feature in features {
feature.extract_aux_frames(&mut aux_frames, threshold);
}
}
aux_frames
}
}
#[cfg(test)]
@@ -144,9 +155,15 @@ mod tests {
use rmpv::Value;
use super::*;
use crate::protocol::multimodal::{
MmBatchedField, MmFeatureSpec, MmField, MmFieldElem, MmKwargValue, PlaceholderRange,
};
use crate::protocol::sampling::EngineCoreSamplingParams;
use crate::protocol::tensor::{WireArrayData, WireTensor};
use crate::protocol::{decode_value, encode_msgpack};
const AUX_FRAME_THRESHOLD: usize = 256;
#[test]
fn engine_core_request_serializes_as_full_array() {
let request = EngineCoreRequest {
@@ -175,4 +192,84 @@ mod tests {
assert_eq!(array[10], Value::Nil);
assert_eq!(array[11], Value::from(7));
}
#[test]
fn engine_core_request_extracts_large_nested_tensors_in_wire_order() {
let inline = vec![1_u8; AUX_FRAME_THRESHOLD - 1];
let first_aux = vec![2_u8; AUX_FRAME_THRESHOLD];
let second_aux = vec![3_u8; AUX_FRAME_THRESHOLD + 1];
let first_aux_ptr = first_aux.as_ptr();
let second_aux_ptr = second_aux.as_ptr();
let mut request = EngineCoreRequest {
mm_features: Some(vec![MmFeatureSpec {
data: Some(BTreeMap::from([
(
"inline".to_string(),
MmFieldElem {
data: Some(MmKwargValue::Tensor(WireTensor::from_raw(
"uint8",
vec![inline.len()],
inline,
))),
field: MmField::Batched(MmBatchedField { keep_on_cpu: false }),
},
),
(
"nested".to_string(),
MmFieldElem {
data: Some(MmKwargValue::List(vec![
MmKwargValue::Int(7),
MmKwargValue::Tensor(WireTensor::from_raw(
"uint8",
vec![first_aux.len()],
first_aux,
)),
])),
field: MmField::Batched(MmBatchedField { keep_on_cpu: false }),
},
),
])),
modality: "image".to_string(),
identifier: "id".to_string(),
mm_position: PlaceholderRange {
offset: 0,
length: second_aux.len(),
is_embed: Some(WireTensor::from_raw(
"bool",
vec![second_aux.len()],
second_aux,
)),
},
mm_hash: None,
}]),
..EngineCoreRequest::default()
};
let aux_frames = request.extract_aux_frames(AUX_FRAME_THRESHOLD);
assert_eq!(aux_frames.len(), 2);
assert_eq!(aux_frames[0].as_ptr(), first_aux_ptr);
assert_eq!(aux_frames[1].as_ptr(), second_aux_ptr);
let feature = &request.mm_features.as_ref().unwrap()[0];
let MmKwargValue::Tensor(inline) =
feature.data.as_ref().unwrap()["inline"].data.as_ref().unwrap()
else {
panic!("expected inline tensor");
};
assert!(matches!(inline.data, WireArrayData::RawView(_)));
let MmKwargValue::List(nested) =
feature.data.as_ref().unwrap()["nested"].data.as_ref().unwrap()
else {
panic!("expected nested tensor list");
};
let MmKwargValue::Tensor(nested_tensor) = &nested[1] else {
panic!("expected nested tensor");
};
assert_eq!(nested_tensor.data, WireArrayData::AuxIndex(1));
assert_eq!(
feature.mm_position.is_embed.as_ref().unwrap().data,
WireArrayData::AuxIndex(2)
);
assert!(request.extract_aux_frames(AUX_FRAME_THRESHOLD).is_empty());
}
}
@@ -1,7 +1,8 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use bytemuck::allocation::pod_collect_to_vec;
use bytemuck::{Pod, cast_slice};
use bytes::Bytes;
use enum_as_inner::EnumAsInner;
use half::{bf16, f16};
use rmpv::Value;
@@ -20,6 +21,21 @@ struct MsgpackExtRef<'a>((i8, ByteSlice<'a>));
struct ByteSlice<'a>(&'a [u8]);
struct PodVec<T: Pod>(Vec<T>);
impl<T: Pod> AsRef<[u8]> for PodVec<T> {
fn as_ref(&self) -> &[u8] {
cast_slice(&self.0)
}
}
fn bytes_from_pod_vec<T>(data: Vec<T>) -> Bytes
where
T: Pod + Send + 'static,
{
Bytes::from_owner(PodVec(data))
}
impl Serialize for ByteSlice<'_> {
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
where
@@ -55,58 +71,63 @@ pub struct WireNdArray {
impl WireNdArray {
/// Build a float32 tensor/ndarray backed by native-endian raw-view bytes.
pub fn from_f32(shape: Vec<usize>, data: impl AsRef<[f32]>) -> Result<Self, String> {
let data = data.as_ref();
///
/// Takes ownership of the backing buffer without copying its data.
pub fn from_f32(shape: Vec<usize>, data: Vec<f32>) -> Result<Self, String> {
validate_element_count(&shape, data.len())?;
Ok(Self {
dtype: "float32".to_string(),
Ok(Self::from_raw_bytes(
"float32",
shape,
data: WireArrayData::RawView(pod_collect_to_vec::<f32, u8>(data)),
})
bytes_from_pod_vec(data),
))
}
/// Build a float16 tensor/ndarray backed by native-endian raw-view bytes.
pub fn from_f16(shape: Vec<usize>, data: impl AsRef<[f16]>) -> Result<Self, String> {
let data = data.as_ref();
///
/// Takes ownership of the backing buffer without copying its data.
pub fn from_f16(shape: Vec<usize>, data: Vec<f16>) -> Result<Self, String> {
validate_element_count(&shape, data.len())?;
Ok(Self {
dtype: "float16".to_string(),
Ok(Self::from_raw_bytes(
"float16",
shape,
data: WireArrayData::RawView(pod_collect_to_vec::<f16, u8>(data)),
})
bytes_from_pod_vec(data),
))
}
/// Build a bfloat16 tensor/ndarray backed by native-endian raw-view bytes.
pub fn from_bf16(shape: Vec<usize>, data: impl AsRef<[bf16]>) -> Result<Self, String> {
let data = data.as_ref();
///
/// Takes ownership of the backing buffer without copying its data.
pub fn from_bf16(shape: Vec<usize>, data: Vec<bf16>) -> Result<Self, String> {
validate_element_count(&shape, data.len())?;
Ok(Self {
dtype: "bfloat16".to_string(),
Ok(Self::from_raw_bytes(
"bfloat16",
shape,
data: WireArrayData::RawView(pod_collect_to_vec::<bf16, u8>(data)),
})
bytes_from_pod_vec(data),
))
}
/// Build an int64 tensor/ndarray backed by native-endian raw-view bytes.
pub fn from_i64(shape: Vec<usize>, data: impl AsRef<[i64]>) -> Result<Self, String> {
let data = data.as_ref();
///
/// Takes ownership of the backing buffer without copying its data.
pub fn from_i64(shape: Vec<usize>, data: Vec<i64>) -> Result<Self, String> {
validate_element_count(&shape, data.len())?;
Ok(Self {
dtype: "int64".to_string(),
Ok(Self::from_raw_bytes(
"int64",
shape,
data: WireArrayData::RawView(pod_collect_to_vec::<i64, u8>(data)),
})
bytes_from_pod_vec(data),
))
}
/// Build a uint32 tensor/ndarray backed by native-endian raw-view bytes.
pub fn from_u32(shape: Vec<usize>, data: impl AsRef<[u32]>) -> Result<Self, String> {
let data = data.as_ref();
///
/// Takes ownership of the backing buffer without copying its data.
pub fn from_u32(shape: Vec<usize>, data: Vec<u32>) -> Result<Self, String> {
validate_element_count(&shape, data.len())?;
Ok(Self {
dtype: "uint32".to_string(),
Ok(Self::from_raw_bytes(
"uint32",
shape,
data: WireArrayData::RawView(pod_collect_to_vec::<u32, u8>(data)),
})
bytes_from_pod_vec(data),
))
}
/// Build a bool tensor/ndarray backed by raw-view bytes.
@@ -118,7 +139,9 @@ impl WireNdArray {
Ok(Self {
dtype: "bool".to_string(),
shape,
data: WireArrayData::RawView(data.iter().map(|value| u8::from(*value)).collect()),
data: WireArrayData::RawView(Bytes::from(
data.into_iter().map(u8::from).collect::<Vec<_>>(),
)),
})
}
@@ -127,12 +150,22 @@ impl WireNdArray {
/// Use this as an escape hatch when the caller already owns bytes that
/// match the requested `dtype` and `shape`.
pub fn from_raw(dtype: impl Into<String>, shape: Vec<usize>, data: Vec<u8>) -> Self {
Self::from_raw_bytes(dtype, shape, Bytes::from(data))
}
/// Build a tensor/ndarray from an owned immutable raw-view buffer.
pub fn from_raw_bytes(dtype: impl Into<String>, shape: Vec<usize>, data: Bytes) -> Self {
Self {
dtype: dtype.into(),
shape,
data: WireArrayData::RawView(data),
}
}
/// Move a sufficiently large inline buffer into the ordered auxiliary-frame list.
pub(crate) fn extract_aux_frame(&mut self, aux_frames: &mut Vec<Bytes>, threshold: usize) {
self.data.extract_aux_frame(aux_frames, threshold);
}
}
/// Validate that the number of elements implied by the shape matches the length
@@ -170,7 +203,25 @@ pub enum WireArrayData {
/// stored.
AuxIndex(usize),
/// The raw bytes of this array/tensor.
RawView(Vec<u8>),
RawView(Bytes),
}
impl WireArrayData {
/// Replace a sufficiently large raw view with its one-based auxiliary-frame index.
fn extract_aux_frame(&mut self, aux_frames: &mut Vec<Bytes>, threshold: usize) {
let Self::RawView(bytes) = self else {
return;
};
if bytes.len() < threshold {
return;
}
let index = aux_frames.len() + 1;
let bytes = std::mem::replace(self, Self::AuxIndex(index))
.into_raw_view()
.expect("raw view was matched above");
aux_frames.push(bytes);
}
}
impl<'de> Deserialize<'de> for WireArrayData {
@@ -180,7 +231,9 @@ impl<'de> Deserialize<'de> for WireArrayData {
{
let value = Value::deserialize(deserializer)?;
match value {
Value::Ext(tag, bytes) if tag == CUSTOM_TYPE_RAW_VIEW => Ok(Self::RawView(bytes)),
Value::Ext(tag, bytes) if tag == CUSTOM_TYPE_RAW_VIEW => {
Ok(Self::RawView(Bytes::from(bytes)))
}
Value::Ext(tag, _) => Err(serde::de::Error::custom(format!(
"unsupported extension type code {tag}"
))),
@@ -201,9 +254,6 @@ impl Serialize for WireArrayData {
where
S: Serializer,
{
// TODO: outbound request serialization currently only supports inline
// raw-view bytes. Emitting aux frames needs transport-level plumbing;
// serializing `AuxIndex` here only preserves an already-built reference.
match self {
Self::AuxIndex(index) => serializer.serialize_u64(*index as u64),
Self::RawView(bytes) => {
@@ -221,7 +271,7 @@ mod tests {
fn raw_view_serializes_as_msgpack_ext() {
let bytes = vec![1, 2, 3, 4];
let encoded =
rmp_serde::to_vec_named(&WireArrayData::RawView(bytes.clone())).expect("encode");
rmp_serde::to_vec_named(&WireArrayData::RawView(bytes.clone().into())).expect("encode");
let expected = rmp_serde::to_vec_named(&Value::Ext(CUSTOM_TYPE_RAW_VIEW, bytes.clone()))
.expect("encode expected");
@@ -234,11 +284,15 @@ mod tests {
#[test]
fn constructors_build_raw_view_tensors() {
let f32_tensor = WireNdArray::from_f32(vec![2], vec![1.0, 2.5]).unwrap();
let f32_data = vec![1.0, 2.5];
let f32_data_ptr = f32_data.as_ptr().cast::<u8>();
let f32_tensor = WireNdArray::from_f32(vec![2], f32_data).unwrap();
assert_eq!(f32_tensor.dtype, "float32");
assert_eq!(f32_tensor.shape, vec![2]);
let f32_raw_view = f32_tensor.data.into_raw_view().expect("raw view");
assert_eq!(f32_raw_view.as_ptr(), f32_data_ptr);
assert_eq!(
f32_tensor.data.into_raw_view().expect("raw view"),
f32_raw_view,
[1.0_f32, 2.5].into_iter().flat_map(f32::to_ne_bytes).collect::<Vec<_>>()
);
@@ -258,15 +312,15 @@ mod tests {
let i64_tensor = WireNdArray::from_i64(vec![1], vec![-7]).unwrap();
assert_eq!(i64_tensor.dtype, "int64");
assert_eq!(
i64_tensor.data.into_raw_view().expect("raw view"),
(-7_i64).to_ne_bytes()
i64_tensor.data.into_raw_view().expect("raw view").as_ref(),
(-7_i64).to_ne_bytes().as_ref()
);
let u32_tensor = WireNdArray::from_u32(vec![1], vec![42]).unwrap();
assert_eq!(u32_tensor.dtype, "uint32");
assert_eq!(
u32_tensor.data.into_raw_view().expect("raw view"),
42_u32.to_ne_bytes()
u32_tensor.data.into_raw_view().expect("raw view").as_ref(),
42_u32.to_ne_bytes().as_ref()
);
let bool_tensor = WireNdArray::from_bool(vec![2], vec![false, true]).unwrap();
@@ -32,7 +32,7 @@ use crate::protocol::output::{
use crate::protocol::request::{EngineCoreRequest, EngineCoreRequestType};
use crate::protocol::sampling::EngineCoreSamplingParams;
use crate::protocol::stats::SchedulerStats;
use crate::protocol::tensor::WireTensor;
use crate::protocol::tensor::{WireArrayData, WireTensor};
use crate::protocol::utility::{UtilityOutput, UtilityResultEnvelope};
use crate::test_utils::{
IpcNamespace, setup_bootstrapped_mock_engine, setup_mock_engine_sockets,
@@ -1728,6 +1728,86 @@ async fn client_decodes_multipart_logprob_outputs() {
client.shutdown().await.unwrap();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn client_sends_large_multimodal_tensor_as_aux_frame() {
init_tracing();
let ipc = IpcNamespace::new().unwrap();
let handshake_address = ipc.handshake_endpoint();
let engine_id = b"engine-multimodal-aux".to_vec();
let tensor_data = (0..64).map(|value| value as f32).collect::<Vec<_>>();
let expected_bytes =
tensor_data.iter().flat_map(|value| value.to_ne_bytes()).collect::<Vec<_>>();
let mut request = sample_multimodal_request();
request.mm_features.as_mut().unwrap()[0]
.data
.as_mut()
.unwrap()
.get_mut("pixel_values")
.unwrap()
.data = Some(MmKwargValue::Tensor(
WireTensor::from_f32(vec![64], tensor_data).unwrap(),
));
let (shutdown_tx, engine_task) = spawn_mock_engine_task(
handshake_address.clone(),
engine_id.clone(),
move |dealer, push| {
Box::pin(async move {
let add = recv_engine_message(dealer).await;
assert_eq!(add.len(), 3);
assert_eq!(add[0].as_ref(), &[0x00]);
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
let MmKwargValue::Tensor(tensor) =
request.mm_features.as_ref().unwrap()[0].data.as_ref().unwrap()["pixel_values"]
.data
.as_ref()
.unwrap()
else {
panic!("expected tensor");
};
assert_eq!(tensor.data, WireArrayData::AuxIndex(1));
assert_eq!(add[2].as_ref(), expected_bytes);
send_outputs(
push,
RequestBatchOutputs {
outputs: vec![request_output(
"req-mm",
vec![],
Some(EngineCoreFinishReason::Length),
)],
finished_requests: Some(BTreeSet::from(["req-mm".to_string()])),
..Default::default()
}
.into(),
)
.await;
})
},
);
let client = connect_client_with_ipc(
handshake_test_config(
handshake_address,
1,
"test-model",
Duration::from_secs(2),
0,
None,
),
&ipc,
)
.await;
let outputs = client.call(request).await.unwrap().collect::<Vec<_>>().await;
assert_eq!(outputs.len(), 1);
assert!(outputs[0].is_ok());
let _ = shutdown_tx.send(());
engine_task.await.unwrap();
client.shutdown().await.unwrap();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn multi_engine_client_shares_transport_and_routes_by_inflight_count() {
init_tracing();
+7 -7
View File
@@ -512,14 +512,14 @@ pub async fn send_message(
input_send: &mut RouterSendHalf,
engine_id: &EngineId,
request_type: Bytes,
payload: Vec<u8>,
payload: Bytes,
aux_frames: Vec<Bytes>,
) -> Result<()> {
let message = ZmqMessage::try_from(vec![
engine_id.to_frame(),
request_type,
Bytes::from(payload),
])
.expect("router messages must contain identity and payload");
let mut frames = Vec::with_capacity(3 + aux_frames.len());
frames.extend([engine_id.to_frame(), request_type, payload]);
frames.extend(aux_frames);
let message =
ZmqMessage::try_from(frames).expect("router messages must contain identity and payload");
trace!(
?engine_id,