[Rust Frontend][Perf] Use dedicated runtime for HTTP/request-processing/ZMQ (#46051)

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-06-23 04:03:20 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 9d3317172c
commit a8481be7a9
13 changed files with 328 additions and 15 deletions
+19 -11
View File
@@ -15,6 +15,7 @@ use crate::protocol::handshake::EngineCoreReadyResponse;
use crate::protocol::lora::LoraRequest;
use crate::protocol::utility::{EngineCoreUtilityRequest, PauseMode};
use crate::protocol::{EngineCoreRequest, EngineCoreRequestType, ModelDtype};
use crate::runtime::{BackgroundShutdownRuntime, build_zmq_runtime};
use crate::transport::{self, ConnectedEngine};
pub(crate) mod imp;
@@ -201,6 +202,8 @@ pub struct EngineCoreClient {
coordinator: Option<CoordinatorHandle>,
abort_tx: mpsc::UnboundedSender<AbortRequest>,
/// Runtime used to send messages to the engine and drive all background tasks.
runtime: BackgroundShutdownRuntime,
// Background tasks
output_task: AbortOnDropHandle<()>,
dispatcher_task: AbortOnDropHandle<()>,
@@ -280,21 +283,22 @@ impl EngineCoreClient {
let (output_tx, output_rx) = mpsc::channel(64);
let (abort_tx, abort_rx) = mpsc::unbounded_channel();
let engines = connected.engines;
let runtime = build_zmq_runtime();
let inner = Arc::new(ClientInner::new(
connected.input_send,
runtime.handle().clone(),
config.model_name.clone(),
&engines,
));
let output_task = AbortOnDropHandle::new(tokio::spawn(transport::run_output_loop(
let output_task = AbortOnDropHandle::new(runtime.spawn(transport::run_output_loop(
connected.output_socket,
output_tx,
)));
let dispatcher_task = AbortOnDropHandle::new(tokio::spawn(run_output_dispatcher_loop(
inner.clone(),
output_rx,
)));
let dispatcher_task = AbortOnDropHandle::new(
runtime.spawn(run_output_dispatcher_loop(inner.clone(), output_rx)),
);
let abort_task =
AbortOnDropHandle::new(tokio::spawn(run_abort_loop(inner.clone(), abort_rx)));
AbortOnDropHandle::new(runtime.spawn(run_abort_loop(inner.clone(), abort_rx)));
// If any engine reported a dp_stats_address in its ready response, use it
// as the external coordinator address.
@@ -307,13 +311,13 @@ impl EngineCoreClient {
CoordinatorHandle::new_inproc(coordinator_transport.input_socket);
let (coordinator_output_tx, coordinator_output_rx) = mpsc::channel(64);
let coordinator_output_task =
AbortOnDropHandle::new(tokio::spawn(transport::run_output_loop(
AbortOnDropHandle::new(runtime.spawn(transport::run_output_loop(
coordinator_transport.output_socket,
coordinator_output_tx,
)));
let coordinator_task = AbortOnDropHandle::new(tokio::spawn(
runner.run(coordinator_output_rx, inner.clone()),
));
let coordinator_task = AbortOnDropHandle::new(
runtime.spawn(runner.run(coordinator_output_rx, inner.clone())),
);
(
Some(handle),
Some(coordinator_output_task),
@@ -327,7 +331,7 @@ impl EngineCoreClient {
{
let (handle, service) = CoordinatorHandle::connect_external(address).await?;
let coordinator_task =
AbortOnDropHandle::new(tokio::spawn(service.run(inner.clone())));
AbortOnDropHandle::new(runtime.spawn(service.run(inner.clone())));
(Some(handle), None, Some(coordinator_task))
} else {
(None, None, None)
@@ -341,6 +345,7 @@ impl EngineCoreClient {
inner,
coordinator,
abort_tx,
runtime,
output_task,
dispatcher_task,
abort_task,
@@ -737,6 +742,7 @@ impl EngineCoreClient {
let Self {
inner,
abort_tx,
runtime,
output_task,
dispatcher_task,
abort_task,
@@ -757,6 +763,8 @@ impl EngineCoreClient {
tasks.iter().for_each(|t| t.abort());
join_all(tasks).await;
drop(inner);
drop(runtime);
info!("engine-core client shut down");
Ok(())
+19 -3
View File
@@ -5,6 +5,7 @@ use std::time::{SystemTime, UNIX_EPOCH};
use arc_swap::ArcSwapOption;
use parking_lot::Mutex;
use thiserror_ext::AsReport as _;
use tokio::runtime::Handle;
use tokio::sync::mpsc;
use tracing::{debug, info, trace, warn};
use vllm_metrics::METRICS;
@@ -26,6 +27,8 @@ use crate::{Error, Result, transport};
pub(crate) struct ClientInner {
input_send: RouterSendHalf,
/// The runtime handle used for sending messages to the engine.
handle: Handle,
model_name: String,
request_reg: Mutex<RequestRegistry>,
utility_reg: Mutex<UtilityRegistry>,
@@ -37,11 +40,13 @@ impl ClientInner {
/// handshake completes.
pub fn new(
input_send: RouterSendHalf,
handle: Handle,
model_name: String,
engines: &[ConnectedEngine],
) -> Self {
Self {
input_send,
handle,
model_name,
request_reg: Mutex::new(RequestRegistry::new(engines)),
utility_reg: Mutex::new(UtilityRegistry::default()),
@@ -213,9 +218,19 @@ impl ClientInner {
// frames instead of always producing a single msgpack frame.
let payload = encode_msgpack(payload)?;
let mut input_send = self.input_send.clone();
transport::send_message(&mut input_send, engine_id, request_type.to_frame(), payload)
.await?;
Ok(())
let engine_id = engine_id.clone();
self.handle
.spawn(async move {
transport::send_message(
&mut input_send,
&engine_id,
request_type.to_frame(),
payload,
)
.await
})
.await?
}
/// Handle an abort request by sending the abort message to the engine.
@@ -434,6 +449,7 @@ mod tests {
let (send, _) = socket.split();
ClientInner::new(
send,
Handle::current(),
"test-model".to_string(),
&[ConnectedEngine {
engine_id: EngineId::from(b"engine-0"),
+2
View File
@@ -29,6 +29,8 @@ pub enum Error {
Io(#[from] std::io::Error),
#[error("transport error")]
Transport(#[from] zeromq::ZmqError),
#[error("ZMQ runtime task failed")]
ZmqRuntimeTask(#[from] tokio::task::JoinError),
#[error("engine core reported fatal failure")]
EngineCoreDead,
#[error("startup handshake timed out while waiting for {stage} after {timeout:?}")]
+1
View File
@@ -4,6 +4,7 @@ mod error;
mod metrics;
pub mod mock_engine;
pub mod protocol;
pub mod runtime;
#[cfg(any(test, feature = "test-util"))]
pub mod test_utils;
mod transport;
@@ -0,0 +1,75 @@
use std::mem::ManuallyDrop;
use std::ops::{Deref, DerefMut};
use std::sync::OnceLock;
use std::sync::atomic::{AtomicUsize, Ordering};
use tokio::runtime::Runtime;
/// A wrapper around [`Runtime`] that shuts down the runtime in the background when dropped.
///
/// This can be useful in some cases, because sometimes we want to drop the runtime without
/// blocking the current thread, for example, when it's nested inside another runtime.
pub struct BackgroundShutdownRuntime(ManuallyDrop<Runtime>);
impl Drop for BackgroundShutdownRuntime {
fn drop(&mut self) {
// Safety: The runtime is only dropped once here.
let runtime = unsafe { ManuallyDrop::take(&mut self.0) };
runtime.shutdown_background();
}
}
impl Deref for BackgroundShutdownRuntime {
type Target = Runtime;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl DerefMut for BackgroundShutdownRuntime {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.0
}
}
impl From<Runtime> for BackgroundShutdownRuntime {
fn from(runtime: Runtime) -> Self {
Self(ManuallyDrop::new(runtime))
}
}
const ZMQ_WORKER_THREADS_ENV: &str = "VLLM_RS_ZMQ_WORKER_THREADS";
/// The number of tasks running on the ZMQ runtime is fixed and expected to remain
/// small, and multiple engines share the same ZMQ socket. Therefore, based on
/// benchmarks, a default value of 4 is generally sufficient.
const DEFAULT_ZMQ_WORKER_THREADS: usize = 4;
static ZMQ_RUNTIME_SEQUENCE: OnceLock<AtomicUsize> = OnceLock::new();
/// Build a Tokio runtime for ZMQ tasks. Multiple calls to this function will
/// return multiple runtimes with distinct thread name suffixes.
pub(crate) fn build_zmq_runtime() -> BackgroundShutdownRuntime {
let sequence = ZMQ_RUNTIME_SEQUENCE
.get_or_init(|| AtomicUsize::new(0))
.fetch_add(1, Ordering::Relaxed);
tokio::runtime::Builder::new_multi_thread()
.worker_threads(zmq_worker_threads())
.thread_name_fn(move || format!("vllm-zmq-{sequence}"))
.enable_all()
.build()
.expect("failed to build vLLM ZMQ runtime")
.into()
}
/// Get the number of worker threads to use for the ZMQ runtime. If env var
/// `VLLM_RS_ZMQ_WORKER_THREADS` is set and a valid positive integer, it will be used.
/// Otherwise, the default value of `DEFAULT_ZMQ_WORKER_THREADS` will be used.
fn zmq_worker_threads() -> usize {
std::env::var(ZMQ_WORKER_THREADS_ENV)
.ok()
.and_then(|value| value.parse::<usize>().ok())
.filter(|value| *value > 0)
.unwrap_or(DEFAULT_ZMQ_WORKER_THREADS)
}