[Rust Frontend][Perf] Use dedicated runtime for HTTP/request-processing/ZMQ (#46051)

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-06-23 04:03:20 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 9d3317172c
commit a8481be7a9
13 changed files with 328 additions and 15 deletions
+1
View File
@@ -30,6 +30,7 @@ tokio-stream.workspace = true
tokio-util.workspace = true
tonic.workspace = true
tonic-prost.workspace = true
tower.workspace = true
tower-http.workspace = true
tracing.workspace = true
tracing-futures.workspace = true
+5 -1
View File
@@ -7,6 +7,7 @@ mod listener;
mod lora;
mod middleware;
mod routes;
mod runtime;
mod server_info;
mod state;
mod utils;
@@ -157,6 +158,9 @@ where
.with_context(|| format!("failed to bind gRPC listener on {grpc_host}:{grpc_port}"))?;
let addr = grpc_listener.local_addr()?;
let svc = grpc::GenerateServer::new(grpc::GenerateServiceImpl::new(state.clone()));
let svc = TonicServer::builder()
.layer(middleware::request_runtime_layer(state.clone()))
.add_service(svc);
info!(%addr, "starting gRPC server");
Some((grpc_listener, svc))
} else {
@@ -238,7 +242,7 @@ where
shutdown.cancelled().await;
return Ok(());
};
let server = TonicServer::builder().add_service(svc).serve_with_incoming_shutdown(
let server = svc.serve_with_incoming_shutdown(
TcpListenerStream::new(grpc_listener),
shutdown.cancelled_owned(),
);
+2
View File
@@ -2,10 +2,12 @@ mod auth;
mod cors;
mod load;
mod metrics;
mod offload;
mod request_id;
pub use auth::authenticate_api_key;
pub use cors::{cors_layer, strip_cors_on_no_origin};
pub use load::track_server_load;
pub use metrics::track_http_metrics;
pub(crate) use offload::request_runtime_layer;
pub use request_id::set_request_id_header;
+134
View File
@@ -0,0 +1,134 @@
use std::sync::Arc;
use std::task::{Context, Poll};
use axum::http::Request;
use axum::response::{IntoResponse, Response};
use futures::future::BoxFuture;
use tokio_util::task::AbortOnDropHandle;
use tonic::Status;
use tower::Service;
use tower::layer::layer_fn;
use tracing::error;
use crate::error::{ApiError, server_error};
use crate::state::AppState;
/// Request paths that are run on the request runtime.
///
/// These routes can perform CPU-heavy request preparation, including JSON
/// extraction, validation, chat-template rendering, tokenization, request
/// lowering, and engine submission. Lightweight operational routes stay on the
/// HTTP runtime.
const OFFLOADED_PATHS: &[&str] = &[
// HTTP routes:
"/v1/chat/completions",
"/v1/completions",
"/tokenize",
"/detokenize",
"/inference/v1/generate",
// gRPC routes:
"/vllm.Generate/Generate",
"/vllm.Generate/GenerateStream",
];
/// Return a Tower layer that runs selected data-plane requests on the request runtime,
/// so that we can offset heavy request parsing and preprocessing from the HTTP runtime.
pub(crate) fn request_runtime_layer<S>(
state: Arc<AppState>,
) -> impl tower::Layer<S, Service = RequestRuntimeService<S>> + Clone {
layer_fn(move |inner| RequestRuntimeService {
inner,
state: state.clone(),
})
}
/// Service produced by [`request_runtime_layer`].
#[derive(Clone)]
pub(crate) struct RequestRuntimeService<S> {
inner: S,
state: Arc<AppState>,
}
impl<S, B> Service<Request<B>> for RequestRuntimeService<S>
where
S: Service<Request<B>> + Clone + Send + 'static,
S::Future: Send + 'static,
S::Response: RequestRuntimeErrorResponse + Send + 'static,
S::Error: Send + 'static,
B: Send + 'static,
{
type Response = S::Response;
type Error = S::Error;
type Future = BoxFuture<'static, Result<Self::Response, Self::Error>>;
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
self.inner.poll_ready(cx)
}
fn call(&mut self, req: Request<B>) -> Self::Future {
if !should_offload(req.uri().path()) {
return Box::pin(self.inner.call(req));
}
// Axum extractors and route handlers execute inside the inner service,
// so offloading here moves request parsing and preprocessing off the
// HTTP runtime without wrapping each handler manually. For streaming
// HTTP responses, the response body is still polled on the HTTP runtime.
let clone = self.inner.clone();
let mut inner = std::mem::replace(&mut self.inner, clone);
let task = AbortOnDropHandle::new(self.state.request_runtime().spawn(inner.call(req)));
Box::pin(async move {
match task.await {
Ok(result) => result,
Err(error) => {
error!(%error, "request runtime task failed");
Ok(S::Response::request_runtime_error_response())
}
}
})
}
}
trait RequestRuntimeErrorResponse {
fn request_runtime_error_response() -> Self;
}
impl RequestRuntimeErrorResponse for Response {
fn request_runtime_error_response() -> Self {
server_error!("request runtime task failed").into_response()
}
}
impl RequestRuntimeErrorResponse for axum::http::Response<tonic::body::Body> {
fn request_runtime_error_response() -> Self {
Status::internal("request runtime task failed").into_http()
}
}
fn should_offload(path: &str) -> bool {
OFFLOADED_PATHS.contains(&path)
}
#[cfg(test)]
mod tests {
use super::should_offload;
#[test]
fn offloads_generation_and_tokenization_paths() {
assert!(should_offload("/v1/chat/completions"));
assert!(should_offload("/v1/completions"));
assert!(should_offload("/tokenize"));
assert!(should_offload("/detokenize"));
assert!(should_offload("/inference/v1/generate"));
assert!(should_offload("/vllm.Generate/Generate"));
assert!(should_offload("/vllm.Generate/GenerateStream"));
}
#[test]
fn passes_through_lightweight_paths() {
assert!(!should_offload("/health"));
assert!(!should_offload("/metrics"));
assert!(!should_offload("/v1/models"));
}
}
+1
View File
@@ -108,6 +108,7 @@ fn build_router_with_options(
let enable_api_key_auth = state.has_api_keys();
let mut router = router
.with_state(state.clone())
.layer(middleware::request_runtime_layer(state.clone()))
.layer(from_fn_with_state(
state.clone(),
middleware::track_server_load,
+54
View File
@@ -0,0 +1,54 @@
use tokio::runtime::Builder;
use tracing::{info, warn};
use vllm_engine_core_client::runtime::BackgroundShutdownRuntime;
const REQUEST_WORKER_THREADS_ENV: &str = "VLLM_RS_REQUEST_WORKER_THREADS";
const DEFAULT_MAX_REQUEST_WORKER_THREADS: usize = 32;
/// Build a Tokio runtime for heavyweight request paths outside the HTTP runtime.
///
/// The server middleware uses this runtime for inference and tokenization
/// routes so CPU-heavy request preparation does not monopolize the HTTP
/// runtime's worker queue. Dropping the wrapper shuts the runtime down in the
/// background.
pub(crate) fn build_request_runtime() -> BackgroundShutdownRuntime {
Builder::new_multi_thread()
.enable_all()
.thread_name("vllm-request")
.worker_threads(request_worker_threads())
.build()
.expect("failed to build request runtime")
.into()
}
/// Get the number of worker threads to use for the request runtime.
///
/// If `VLLM_RS_REQUEST_WORKER_THREADS` is set to a valid positive integer, it is
/// used directly. Otherwise, the runtime uses available parallelism capped by
/// `DEFAULT_MAX_REQUEST_WORKER_THREADS`.
fn request_worker_threads() -> usize {
if let Some(value) = std::env::var_os(REQUEST_WORKER_THREADS_ENV) {
match value.to_string_lossy().parse::<usize>() {
Ok(worker_threads) if worker_threads > 0 => return worker_threads,
_ => warn!(
value = %value.to_string_lossy(),
"ignoring invalid {REQUEST_WORKER_THREADS_ENV}"
),
}
}
std::thread::available_parallelism()
.map(|parallelism| {
let available = parallelism.get();
let worker_threads = available.min(DEFAULT_MAX_REQUEST_WORKER_THREADS);
if worker_threads < available {
info!(
available_parallelism = available,
capped_worker_threads = worker_threads,
"capping request runtime worker threads, set {REQUEST_WORKER_THREADS_ENV} to override"
);
}
worker_threads
})
.unwrap_or(DEFAULT_MAX_REQUEST_WORKER_THREADS)
}
+14
View File
@@ -1,16 +1,20 @@
use std::sync::Arc;
use std::sync::OnceLock;
use std::sync::atomic::{AtomicU64, Ordering};
use serde_json::Value;
use sha2::{Digest, Sha256};
use tokio::runtime::Runtime;
use tokio::time::{Duration, Instant, sleep_until};
use tracing::warn;
use vllm_chat::ChatLlm;
use vllm_engine_core_client::EngineCoreClient;
use vllm_engine_core_client::protocol::lora::LoraRequest;
use vllm_engine_core_client::runtime::BackgroundShutdownRuntime;
use crate::config::{ApiServerOptions, CorsConfig};
use crate::lora::{LoadLoraError, LoraManager, LoraModelResolution, UnloadLoraError};
use crate::runtime::build_request_runtime;
use crate::server_info::{ServerInfoConfigFormat, ServerInfoSnapshot};
const SHUTDOWN_REFCOUNT_POLL_INTERVAL: Duration = Duration::from_millis(100);
@@ -42,6 +46,8 @@ pub struct AppState {
lora_manager: LoraManager,
/// Backend model path reported as `root` for base-model cards.
model_path: Option<String>,
/// Lazily initialized runtime for heavyweight request paths.
request_runtime: OnceLock<BackgroundShutdownRuntime>,
}
impl AppState {
@@ -68,6 +74,7 @@ impl AppState {
server_load: AtomicU64::new(0),
lora_manager: LoraManager::new(),
model_path: None,
request_runtime: OnceLock::new(),
}
}
@@ -185,6 +192,12 @@ impl AppState {
self.chat.engine_core_client()
}
/// Runtime used by middleware to isolate heavyweight request handlers from
/// the HTTP reactor.
pub(crate) fn request_runtime(&self) -> &Runtime {
self.request_runtime.get_or_init(build_request_runtime)
}
/// Return the current in-flight inference request count for the `/load`
/// endpoint.
pub fn server_load(&self) -> u64 {
@@ -214,6 +227,7 @@ impl AppState {
match Arc::try_unwrap(self) {
Ok(state) => {
state.chat.shutdown().await?;
drop(state.request_runtime); // shutdown in background
return Ok(());
}
Err(state) => self = state,