[Rust Frontend] Add server router extension hook (#43774)

Signed-off-by: NolanHo <kujyo.eia.serias@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
NolanHo
2026-06-03 07:45:31 +00:00
committed by GitHub
co-authored by OpenAI Codex Bugen Zhao
parent 7268457999
commit 4aaed4ca22
+17 -2
View File
@@ -12,7 +12,7 @@ mod utils;
use std::sync::{Arc, OnceLock};
use anyhow::{Context as _, Result};
use axum::serve::ListenerExt as _;
use axum::{Router, serve::ListenerExt as _};
pub use config::{Config, CoordinatorMode, HttpListenerMode};
use tokio::net::TcpListener;
use tokio::time::{Instant, sleep_until};
@@ -97,6 +97,21 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
/// The server owns one `vllm-chat` facade, which in turn owns the lower
/// `vllm-text` and `vllm-llm` layers, and shuts them down before returning.
pub async fn serve(config: Config, shutdown: CancellationToken) -> Result<()> {
serve_with_router_extension(config, shutdown, |router| router).await
}
/// Run the OpenAI-compatible HTTP server with an opt-in router extension.
///
/// The extension receives the finalized vLLM router and can merge additional
/// routes before the server starts accepting requests.
pub async fn serve_with_router_extension<F>(
config: Config,
shutdown: CancellationToken,
extend_router: F,
) -> Result<()>
where
F: FnOnce(Router) -> Router,
{
config.validate().context("invalid OpenAI frontend configuration")?;
// Also check shutdown during the (potentially long) startup handshake.
@@ -109,7 +124,7 @@ pub async fn serve(config: Config, shutdown: CancellationToken) -> Result<()> {
.context("failed to bind listener for OpenAI server")?;
let bind_address = listener.local_addr()?;
let model = state.primary_model_name().to_owned();
let app = build_router(state.clone());
let app = extend_router(build_router(state.clone()));
// Optionally bind the gRPC Generate server on a separate port. Bind
// synchronously here so bind errors (port in use, permission denied, ...)