[Frontend] Consolidate entrypoint middleware (#52309)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
wang.yuqi
2026-08-17 06:17:15 +00:00
committed by GitHub
parent 53e211d292
commit 93550cc4cd
14 changed files with 567 additions and 370 deletions
+5 -114
View File
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import importlib
import inspect
import multiprocessing
import multiprocessing.forkserver as forkserver
import os
@@ -17,7 +15,6 @@ from typing import Any, cast
import uvloop
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from starlette.datastructures import State
import vllm.envs as envs
@@ -26,11 +23,12 @@ from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.chat_utils import load_chat_template
from vllm.entrypoints.launcher import serve_http
from vllm.entrypoints.launchers.api_server.routers import register_api_routers
from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.elastic_ep.middleware import ScalingMiddleware
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
from vllm.entrypoints.serve.middleware.register import init_entrypoints_middleware
from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
from vllm.entrypoints.serve.utils.api_utils import (
@@ -43,7 +41,6 @@ from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.entrypoints.serve.utils.server_utils import (
get_uvicorn_log_config,
lifespan,
log_response,
)
from vllm.logger import init_logger
from vllm.reasoning import ReasoningParserManager
@@ -205,64 +202,9 @@ def build_app(
else:
app = FastAPI(lifespan=lifespan)
app.state.args = args
app.root_path = args.root_path
from vllm.entrypoints.serve import register_vllm_serve_api_routers
register_vllm_serve_api_routers(app)
from vllm.entrypoints.openai.models.api_router import (
attach_router as register_models_api_router,
)
register_models_api_router(app)
from vllm.entrypoints.serve.sagemaker.api_router import (
attach_router as register_sagemaker_api_router,
)
register_sagemaker_api_router(app, supported_tasks, model_config)
if envs.VLLM_SERVER_DEV_MODE:
from vllm.entrypoints.serve import register_vllm_dev_api_routers
register_vllm_dev_api_routers(app)
if "generate" in supported_tasks:
from vllm.entrypoints.generate.api_router import (
register_generate_api_routers,
)
register_generate_api_routers(app)
from vllm.entrypoints.serve.elastic_ep.api_router import (
attach_router as elastic_ep_attach_router,
)
elastic_ep_attach_router(app)
if "generate" in supported_tasks or "render" in supported_tasks:
from vllm.entrypoints.scale_out.factories import register_scale_out_api_routers
register_scale_out_api_routers(app, supported_tasks)
if "transcription" in supported_tasks or "realtime" in supported_tasks:
from vllm.entrypoints.speech_to_text.factories import (
register_speech_to_text_api_routers,
)
register_speech_to_text_api_routers(app, supported_tasks)
if any(task in POOLING_TASKS for task in supported_tasks):
from vllm.entrypoints.pooling.factories import register_pooling_api_routers
register_pooling_api_routers(app, supported_tasks, model_config)
if args.enable_fault_tolerance:
from vllm.entrypoints.serve.fault_tolerance.api_router import (
register_fault_tolerance_api_router,
)
register_fault_tolerance_api_router(app)
register_api_routers(args, app, supported_tasks, model_config)
# Endpoint plugins are attached last so their routes are registered after all core
# routers. This runs even for the CPU only render server. A plugin eligible for
@@ -271,58 +213,7 @@ def build_app(
_attach_endpoint_plugins(app, supported_tasks)
init_exception_handler(app)
app.root_path = args.root_path
app.add_middleware(
CORSMiddleware,
allow_origins=args.allowed_origins,
allow_credentials=args.allow_credentials,
allow_methods=args.allowed_methods,
allow_headers=args.allowed_headers,
)
# Ensure --api-key option from CLI takes precedence over VLLM_API_KEY
if tokens := [key for key in (args.api_key or [envs.VLLM_API_KEY]) if key]:
from vllm.entrypoints.serve.utils.server_utils import AuthenticationMiddleware
app.add_middleware(AuthenticationMiddleware, tokens=tokens)
if args.enable_request_id_headers:
from vllm.entrypoints.serve.utils.server_utils import XRequestIdMiddleware
app.add_middleware(XRequestIdMiddleware)
# Add scaling middleware to check for scaling state
app.add_middleware(ScalingMiddleware)
if "realtime" in supported_tasks:
# Add WebSocket metrics middleware
from vllm.entrypoints.speech_to_text.factories import (
add_websocket_metrics_middleware,
)
add_websocket_metrics_middleware(app)
if envs.VLLM_DEBUG_LOG_API_SERVER_RESPONSE:
logger.warning(
"CAUTION: Enabling log response in the API Server. "
"This can include sensitive information and should be "
"avoided in production."
)
app.middleware("http")(log_response)
for middleware in args.middleware:
module_path, object_name = middleware.rsplit(".", 1)
imported = getattr(importlib.import_module(module_path), object_name)
if inspect.isclass(imported):
app.add_middleware(imported) # type: ignore[arg-type]
elif inspect.iscoroutinefunction(imported):
app.middleware("http")(imported)
else:
raise ValueError(
f"Invalid middleware {middleware}. Must be a function or a class."
)
init_entrypoints_middleware(args, app, supported_tasks)
app = sagemaker_standards_bootstrap(app)
return app