[Rust Frontend][Perf] Use dedicated runtime for HTTP/request-processing/ZMQ (#46051)
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
9d3317172c
commit
a8481be7a9
@@ -15,6 +15,7 @@ use crate::protocol::handshake::EngineCoreReadyResponse;
|
||||
use crate::protocol::lora::LoraRequest;
|
||||
use crate::protocol::utility::{EngineCoreUtilityRequest, PauseMode};
|
||||
use crate::protocol::{EngineCoreRequest, EngineCoreRequestType, ModelDtype};
|
||||
use crate::runtime::{BackgroundShutdownRuntime, build_zmq_runtime};
|
||||
use crate::transport::{self, ConnectedEngine};
|
||||
|
||||
pub(crate) mod imp;
|
||||
@@ -201,6 +202,8 @@ pub struct EngineCoreClient {
|
||||
coordinator: Option<CoordinatorHandle>,
|
||||
abort_tx: mpsc::UnboundedSender<AbortRequest>,
|
||||
|
||||
/// Runtime used to send messages to the engine and drive all background tasks.
|
||||
runtime: BackgroundShutdownRuntime,
|
||||
// Background tasks
|
||||
output_task: AbortOnDropHandle<()>,
|
||||
dispatcher_task: AbortOnDropHandle<()>,
|
||||
@@ -280,21 +283,22 @@ impl EngineCoreClient {
|
||||
let (output_tx, output_rx) = mpsc::channel(64);
|
||||
let (abort_tx, abort_rx) = mpsc::unbounded_channel();
|
||||
let engines = connected.engines;
|
||||
let runtime = build_zmq_runtime();
|
||||
let inner = Arc::new(ClientInner::new(
|
||||
connected.input_send,
|
||||
runtime.handle().clone(),
|
||||
config.model_name.clone(),
|
||||
&engines,
|
||||
));
|
||||
let output_task = AbortOnDropHandle::new(tokio::spawn(transport::run_output_loop(
|
||||
let output_task = AbortOnDropHandle::new(runtime.spawn(transport::run_output_loop(
|
||||
connected.output_socket,
|
||||
output_tx,
|
||||
)));
|
||||
let dispatcher_task = AbortOnDropHandle::new(tokio::spawn(run_output_dispatcher_loop(
|
||||
inner.clone(),
|
||||
output_rx,
|
||||
)));
|
||||
let dispatcher_task = AbortOnDropHandle::new(
|
||||
runtime.spawn(run_output_dispatcher_loop(inner.clone(), output_rx)),
|
||||
);
|
||||
let abort_task =
|
||||
AbortOnDropHandle::new(tokio::spawn(run_abort_loop(inner.clone(), abort_rx)));
|
||||
AbortOnDropHandle::new(runtime.spawn(run_abort_loop(inner.clone(), abort_rx)));
|
||||
|
||||
// If any engine reported a dp_stats_address in its ready response, use it
|
||||
// as the external coordinator address.
|
||||
@@ -307,13 +311,13 @@ impl EngineCoreClient {
|
||||
CoordinatorHandle::new_inproc(coordinator_transport.input_socket);
|
||||
let (coordinator_output_tx, coordinator_output_rx) = mpsc::channel(64);
|
||||
let coordinator_output_task =
|
||||
AbortOnDropHandle::new(tokio::spawn(transport::run_output_loop(
|
||||
AbortOnDropHandle::new(runtime.spawn(transport::run_output_loop(
|
||||
coordinator_transport.output_socket,
|
||||
coordinator_output_tx,
|
||||
)));
|
||||
let coordinator_task = AbortOnDropHandle::new(tokio::spawn(
|
||||
runner.run(coordinator_output_rx, inner.clone()),
|
||||
));
|
||||
let coordinator_task = AbortOnDropHandle::new(
|
||||
runtime.spawn(runner.run(coordinator_output_rx, inner.clone())),
|
||||
);
|
||||
(
|
||||
Some(handle),
|
||||
Some(coordinator_output_task),
|
||||
@@ -327,7 +331,7 @@ impl EngineCoreClient {
|
||||
{
|
||||
let (handle, service) = CoordinatorHandle::connect_external(address).await?;
|
||||
let coordinator_task =
|
||||
AbortOnDropHandle::new(tokio::spawn(service.run(inner.clone())));
|
||||
AbortOnDropHandle::new(runtime.spawn(service.run(inner.clone())));
|
||||
(Some(handle), None, Some(coordinator_task))
|
||||
} else {
|
||||
(None, None, None)
|
||||
@@ -341,6 +345,7 @@ impl EngineCoreClient {
|
||||
inner,
|
||||
coordinator,
|
||||
abort_tx,
|
||||
runtime,
|
||||
output_task,
|
||||
dispatcher_task,
|
||||
abort_task,
|
||||
@@ -737,6 +742,7 @@ impl EngineCoreClient {
|
||||
let Self {
|
||||
inner,
|
||||
abort_tx,
|
||||
runtime,
|
||||
output_task,
|
||||
dispatcher_task,
|
||||
abort_task,
|
||||
@@ -757,6 +763,8 @@ impl EngineCoreClient {
|
||||
|
||||
tasks.iter().for_each(|t| t.abort());
|
||||
join_all(tasks).await;
|
||||
drop(inner);
|
||||
drop(runtime);
|
||||
|
||||
info!("engine-core client shut down");
|
||||
Ok(())
|
||||
|
||||
@@ -5,6 +5,7 @@ use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use arc_swap::ArcSwapOption;
|
||||
use parking_lot::Mutex;
|
||||
use thiserror_ext::AsReport as _;
|
||||
use tokio::runtime::Handle;
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::{debug, info, trace, warn};
|
||||
use vllm_metrics::METRICS;
|
||||
@@ -26,6 +27,8 @@ use crate::{Error, Result, transport};
|
||||
|
||||
pub(crate) struct ClientInner {
|
||||
input_send: RouterSendHalf,
|
||||
/// The runtime handle used for sending messages to the engine.
|
||||
handle: Handle,
|
||||
model_name: String,
|
||||
request_reg: Mutex<RequestRegistry>,
|
||||
utility_reg: Mutex<UtilityRegistry>,
|
||||
@@ -37,11 +40,13 @@ impl ClientInner {
|
||||
/// handshake completes.
|
||||
pub fn new(
|
||||
input_send: RouterSendHalf,
|
||||
handle: Handle,
|
||||
model_name: String,
|
||||
engines: &[ConnectedEngine],
|
||||
) -> Self {
|
||||
Self {
|
||||
input_send,
|
||||
handle,
|
||||
model_name,
|
||||
request_reg: Mutex::new(RequestRegistry::new(engines)),
|
||||
utility_reg: Mutex::new(UtilityRegistry::default()),
|
||||
@@ -213,9 +218,19 @@ impl ClientInner {
|
||||
// frames instead of always producing a single msgpack frame.
|
||||
let payload = encode_msgpack(payload)?;
|
||||
let mut input_send = self.input_send.clone();
|
||||
transport::send_message(&mut input_send, engine_id, request_type.to_frame(), payload)
|
||||
.await?;
|
||||
Ok(())
|
||||
let engine_id = engine_id.clone();
|
||||
|
||||
self.handle
|
||||
.spawn(async move {
|
||||
transport::send_message(
|
||||
&mut input_send,
|
||||
&engine_id,
|
||||
request_type.to_frame(),
|
||||
payload,
|
||||
)
|
||||
.await
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
/// Handle an abort request by sending the abort message to the engine.
|
||||
@@ -434,6 +449,7 @@ mod tests {
|
||||
let (send, _) = socket.split();
|
||||
ClientInner::new(
|
||||
send,
|
||||
Handle::current(),
|
||||
"test-model".to_string(),
|
||||
&[ConnectedEngine {
|
||||
engine_id: EngineId::from(b"engine-0"),
|
||||
|
||||
@@ -29,6 +29,8 @@ pub enum Error {
|
||||
Io(#[from] std::io::Error),
|
||||
#[error("transport error")]
|
||||
Transport(#[from] zeromq::ZmqError),
|
||||
#[error("ZMQ runtime task failed")]
|
||||
ZmqRuntimeTask(#[from] tokio::task::JoinError),
|
||||
#[error("engine core reported fatal failure")]
|
||||
EngineCoreDead,
|
||||
#[error("startup handshake timed out while waiting for {stage} after {timeout:?}")]
|
||||
|
||||
@@ -4,6 +4,7 @@ mod error;
|
||||
mod metrics;
|
||||
pub mod mock_engine;
|
||||
pub mod protocol;
|
||||
pub mod runtime;
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub mod test_utils;
|
||||
mod transport;
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
use std::mem::ManuallyDrop;
|
||||
use std::ops::{Deref, DerefMut};
|
||||
use std::sync::OnceLock;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
use tokio::runtime::Runtime;
|
||||
|
||||
/// A wrapper around [`Runtime`] that shuts down the runtime in the background when dropped.
|
||||
///
|
||||
/// This can be useful in some cases, because sometimes we want to drop the runtime without
|
||||
/// blocking the current thread, for example, when it's nested inside another runtime.
|
||||
pub struct BackgroundShutdownRuntime(ManuallyDrop<Runtime>);
|
||||
|
||||
impl Drop for BackgroundShutdownRuntime {
|
||||
fn drop(&mut self) {
|
||||
// Safety: The runtime is only dropped once here.
|
||||
let runtime = unsafe { ManuallyDrop::take(&mut self.0) };
|
||||
runtime.shutdown_background();
|
||||
}
|
||||
}
|
||||
|
||||
impl Deref for BackgroundShutdownRuntime {
|
||||
type Target = Runtime;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl DerefMut for BackgroundShutdownRuntime {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
&mut self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Runtime> for BackgroundShutdownRuntime {
|
||||
fn from(runtime: Runtime) -> Self {
|
||||
Self(ManuallyDrop::new(runtime))
|
||||
}
|
||||
}
|
||||
|
||||
const ZMQ_WORKER_THREADS_ENV: &str = "VLLM_RS_ZMQ_WORKER_THREADS";
|
||||
/// The number of tasks running on the ZMQ runtime is fixed and expected to remain
|
||||
/// small, and multiple engines share the same ZMQ socket. Therefore, based on
|
||||
/// benchmarks, a default value of 4 is generally sufficient.
|
||||
const DEFAULT_ZMQ_WORKER_THREADS: usize = 4;
|
||||
|
||||
static ZMQ_RUNTIME_SEQUENCE: OnceLock<AtomicUsize> = OnceLock::new();
|
||||
|
||||
/// Build a Tokio runtime for ZMQ tasks. Multiple calls to this function will
|
||||
/// return multiple runtimes with distinct thread name suffixes.
|
||||
pub(crate) fn build_zmq_runtime() -> BackgroundShutdownRuntime {
|
||||
let sequence = ZMQ_RUNTIME_SEQUENCE
|
||||
.get_or_init(|| AtomicUsize::new(0))
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
tokio::runtime::Builder::new_multi_thread()
|
||||
.worker_threads(zmq_worker_threads())
|
||||
.thread_name_fn(move || format!("vllm-zmq-{sequence}"))
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("failed to build vLLM ZMQ runtime")
|
||||
.into()
|
||||
}
|
||||
|
||||
/// Get the number of worker threads to use for the ZMQ runtime. If env var
|
||||
/// `VLLM_RS_ZMQ_WORKER_THREADS` is set and a valid positive integer, it will be used.
|
||||
/// Otherwise, the default value of `DEFAULT_ZMQ_WORKER_THREADS` will be used.
|
||||
fn zmq_worker_threads() -> usize {
|
||||
std::env::var(ZMQ_WORKER_THREADS_ENV)
|
||||
.ok()
|
||||
.and_then(|value| value.parse::<usize>().ok())
|
||||
.filter(|value| *value > 0)
|
||||
.unwrap_or(DEFAULT_ZMQ_WORKER_THREADS)
|
||||
}
|
||||
Reference in New Issue
Block a user