diff --git a/rust/src/server/src/lib.rs b/rust/src/server/src/lib.rs index 67d9a8a73a..d051c6db24 100644 --- a/rust/src/server/src/lib.rs +++ b/rust/src/server/src/lib.rs @@ -12,7 +12,7 @@ mod utils; use std::sync::{Arc, OnceLock}; use anyhow::{Context as _, Result}; -use axum::serve::ListenerExt as _; +use axum::{Router, serve::ListenerExt as _}; pub use config::{Config, CoordinatorMode, HttpListenerMode}; use tokio::net::TcpListener; use tokio::time::{Instant, sleep_until}; @@ -97,6 +97,21 @@ async fn build_state(config: &Config) -> Result> { /// The server owns one `vllm-chat` facade, which in turn owns the lower /// `vllm-text` and `vllm-llm` layers, and shuts them down before returning. pub async fn serve(config: Config, shutdown: CancellationToken) -> Result<()> { + serve_with_router_extension(config, shutdown, |router| router).await +} + +/// Run the OpenAI-compatible HTTP server with an opt-in router extension. +/// +/// The extension receives the finalized vLLM router and can merge additional +/// routes before the server starts accepting requests. +pub async fn serve_with_router_extension( + config: Config, + shutdown: CancellationToken, + extend_router: F, +) -> Result<()> +where + F: FnOnce(Router) -> Router, +{ config.validate().context("invalid OpenAI frontend configuration")?; // Also check shutdown during the (potentially long) startup handshake. @@ -109,7 +124,7 @@ pub async fn serve(config: Config, shutdown: CancellationToken) -> Result<()> { .context("failed to bind listener for OpenAI server")?; let bind_address = listener.local_addr()?; let model = state.primary_model_name().to_owned(); - let app = build_router(state.clone()); + let app = extend_router(build_router(state.clone())); // Optionally bind the gRPC Generate server on a separate port. Bind // synchronously here so bind errors (port in use, permission denied, ...)