[Rust Frontend] Add server router extension hook (#43774)
Signed-off-by: NolanHo <kujyo.eia.serias@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
OpenAI Codex
Bugen Zhao
parent
7268457999
commit
4aaed4ca22
@@ -12,7 +12,7 @@ mod utils;
|
||||
use std::sync::{Arc, OnceLock};
|
||||
|
||||
use anyhow::{Context as _, Result};
|
||||
use axum::serve::ListenerExt as _;
|
||||
use axum::{Router, serve::ListenerExt as _};
|
||||
pub use config::{Config, CoordinatorMode, HttpListenerMode};
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::time::{Instant, sleep_until};
|
||||
@@ -97,6 +97,21 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
|
||||
/// The server owns one `vllm-chat` facade, which in turn owns the lower
|
||||
/// `vllm-text` and `vllm-llm` layers, and shuts them down before returning.
|
||||
pub async fn serve(config: Config, shutdown: CancellationToken) -> Result<()> {
|
||||
serve_with_router_extension(config, shutdown, |router| router).await
|
||||
}
|
||||
|
||||
/// Run the OpenAI-compatible HTTP server with an opt-in router extension.
|
||||
///
|
||||
/// The extension receives the finalized vLLM router and can merge additional
|
||||
/// routes before the server starts accepting requests.
|
||||
pub async fn serve_with_router_extension<F>(
|
||||
config: Config,
|
||||
shutdown: CancellationToken,
|
||||
extend_router: F,
|
||||
) -> Result<()>
|
||||
where
|
||||
F: FnOnce(Router) -> Router,
|
||||
{
|
||||
config.validate().context("invalid OpenAI frontend configuration")?;
|
||||
|
||||
// Also check shutdown during the (potentially long) startup handshake.
|
||||
@@ -109,7 +124,7 @@ pub async fn serve(config: Config, shutdown: CancellationToken) -> Result<()> {
|
||||
.context("failed to bind listener for OpenAI server")?;
|
||||
let bind_address = listener.local_addr()?;
|
||||
let model = state.primary_model_name().to_owned();
|
||||
let app = build_router(state.clone());
|
||||
let app = extend_router(build_router(state.clone()));
|
||||
|
||||
// Optionally bind the gRPC Generate server on a separate port. Bind
|
||||
// synchronously here so bind errors (port in use, permission denied, ...)
|
||||
|
||||
Reference in New Issue
Block a user