[Frontend] Consolidate entrypoint middleware (#52309)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import importlib
|
||||
import inspect
|
||||
import multiprocessing
|
||||
import multiprocessing.forkserver as forkserver
|
||||
import os
|
||||
@@ -17,7 +15,6 @@ from typing import Any, cast
|
||||
|
||||
import uvloop
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from starlette.datastructures import State
|
||||
|
||||
import vllm.envs as envs
|
||||
@@ -26,11 +23,12 @@ from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.chat_utils import load_chat_template
|
||||
from vllm.entrypoints.launcher import serve_http
|
||||
from vllm.entrypoints.launchers.api_server.routers import register_api_routers
|
||||
from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.elastic_ep.middleware import ScalingMiddleware
|
||||
from vllm.entrypoints.serve.exception_handling.register import init_exception_handler
|
||||
from vllm.entrypoints.serve.middleware.register import init_entrypoints_middleware
|
||||
from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap
|
||||
from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
@@ -43,7 +41,6 @@ from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.entrypoints.serve.utils.server_utils import (
|
||||
get_uvicorn_log_config,
|
||||
lifespan,
|
||||
log_response,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.reasoning import ReasoningParserManager
|
||||
@@ -205,64 +202,9 @@ def build_app(
|
||||
else:
|
||||
app = FastAPI(lifespan=lifespan)
|
||||
app.state.args = args
|
||||
app.root_path = args.root_path
|
||||
|
||||
from vllm.entrypoints.serve import register_vllm_serve_api_routers
|
||||
|
||||
register_vllm_serve_api_routers(app)
|
||||
|
||||
from vllm.entrypoints.openai.models.api_router import (
|
||||
attach_router as register_models_api_router,
|
||||
)
|
||||
|
||||
register_models_api_router(app)
|
||||
|
||||
from vllm.entrypoints.serve.sagemaker.api_router import (
|
||||
attach_router as register_sagemaker_api_router,
|
||||
)
|
||||
|
||||
register_sagemaker_api_router(app, supported_tasks, model_config)
|
||||
|
||||
if envs.VLLM_SERVER_DEV_MODE:
|
||||
from vllm.entrypoints.serve import register_vllm_dev_api_routers
|
||||
|
||||
register_vllm_dev_api_routers(app)
|
||||
|
||||
if "generate" in supported_tasks:
|
||||
from vllm.entrypoints.generate.api_router import (
|
||||
register_generate_api_routers,
|
||||
)
|
||||
|
||||
register_generate_api_routers(app)
|
||||
|
||||
from vllm.entrypoints.serve.elastic_ep.api_router import (
|
||||
attach_router as elastic_ep_attach_router,
|
||||
)
|
||||
|
||||
elastic_ep_attach_router(app)
|
||||
|
||||
if "generate" in supported_tasks or "render" in supported_tasks:
|
||||
from vllm.entrypoints.scale_out.factories import register_scale_out_api_routers
|
||||
|
||||
register_scale_out_api_routers(app, supported_tasks)
|
||||
|
||||
if "transcription" in supported_tasks or "realtime" in supported_tasks:
|
||||
from vllm.entrypoints.speech_to_text.factories import (
|
||||
register_speech_to_text_api_routers,
|
||||
)
|
||||
|
||||
register_speech_to_text_api_routers(app, supported_tasks)
|
||||
|
||||
if any(task in POOLING_TASKS for task in supported_tasks):
|
||||
from vllm.entrypoints.pooling.factories import register_pooling_api_routers
|
||||
|
||||
register_pooling_api_routers(app, supported_tasks, model_config)
|
||||
|
||||
if args.enable_fault_tolerance:
|
||||
from vllm.entrypoints.serve.fault_tolerance.api_router import (
|
||||
register_fault_tolerance_api_router,
|
||||
)
|
||||
|
||||
register_fault_tolerance_api_router(app)
|
||||
register_api_routers(args, app, supported_tasks, model_config)
|
||||
|
||||
# Endpoint plugins are attached last so their routes are registered after all core
|
||||
# routers. This runs even for the CPU only render server. A plugin eligible for
|
||||
@@ -271,58 +213,7 @@ def build_app(
|
||||
_attach_endpoint_plugins(app, supported_tasks)
|
||||
|
||||
init_exception_handler(app)
|
||||
|
||||
app.root_path = args.root_path
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=args.allowed_origins,
|
||||
allow_credentials=args.allow_credentials,
|
||||
allow_methods=args.allowed_methods,
|
||||
allow_headers=args.allowed_headers,
|
||||
)
|
||||
|
||||
# Ensure --api-key option from CLI takes precedence over VLLM_API_KEY
|
||||
if tokens := [key for key in (args.api_key or [envs.VLLM_API_KEY]) if key]:
|
||||
from vllm.entrypoints.serve.utils.server_utils import AuthenticationMiddleware
|
||||
|
||||
app.add_middleware(AuthenticationMiddleware, tokens=tokens)
|
||||
|
||||
if args.enable_request_id_headers:
|
||||
from vllm.entrypoints.serve.utils.server_utils import XRequestIdMiddleware
|
||||
|
||||
app.add_middleware(XRequestIdMiddleware)
|
||||
|
||||
# Add scaling middleware to check for scaling state
|
||||
app.add_middleware(ScalingMiddleware)
|
||||
|
||||
if "realtime" in supported_tasks:
|
||||
# Add WebSocket metrics middleware
|
||||
from vllm.entrypoints.speech_to_text.factories import (
|
||||
add_websocket_metrics_middleware,
|
||||
)
|
||||
|
||||
add_websocket_metrics_middleware(app)
|
||||
|
||||
if envs.VLLM_DEBUG_LOG_API_SERVER_RESPONSE:
|
||||
logger.warning(
|
||||
"CAUTION: Enabling log response in the API Server. "
|
||||
"This can include sensitive information and should be "
|
||||
"avoided in production."
|
||||
)
|
||||
app.middleware("http")(log_response)
|
||||
|
||||
for middleware in args.middleware:
|
||||
module_path, object_name = middleware.rsplit(".", 1)
|
||||
imported = getattr(importlib.import_module(module_path), object_name)
|
||||
if inspect.isclass(imported):
|
||||
app.add_middleware(imported) # type: ignore[arg-type]
|
||||
elif inspect.iscoroutinefunction(imported):
|
||||
app.middleware("http")(imported)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Invalid middleware {middleware}. Must be a function or a class."
|
||||
)
|
||||
|
||||
init_entrypoints_middleware(args, app, supported_tasks)
|
||||
app = sagemaker_standards_bootstrap(app)
|
||||
return app
|
||||
|
||||
|
||||
Reference in New Issue
Block a user