[Frontend] Add endpoint plugins framework (#47454)

Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
Martin Hickey
2026-07-08 10:00:41 +08:00
committed by GitHub
parent f7efab58ec
commit f7fc0ca993
12 changed files with 660 additions and 1 deletions
+47
View File
@@ -74,6 +74,41 @@ logger = init_logger("vllm.entrypoints.openai.api_server")
_FALLBACK_SUPPORTED_TASKS: tuple[SupportedTask, ...] = ("generate",)
def _attach_endpoint_plugins(
app: FastAPI, supported_tasks: tuple["SupportedTask", ...]
) -> None:
"""Phase A of endpoint plugin wiring: discover, gate and attach routes.
Attached last after all core routers. This is so endpoint plugin routes can
shadow core routes with the same path (see `EndpointPlugin.attach_router`
docstring). No-ops when no plugins are discovered/allowlisted.
"""
from vllm.plugins import load_endpoint_plugins
endpoint_plugins = load_endpoint_plugins(supported_tasks)
for plugin in endpoint_plugins:
plugin.attach_router(app)
app.state.endpoint_plugins = endpoint_plugins
async def _init_endpoint_plugins_state(
engine_client: EngineClient | None, state: State, args: Namespace
) -> None:
"""Phase B of endpoint plugin wiring: initialize per app plugin state.
`state.endpoint_plugins` is set by `_attach_endpoint_plugins` (Phase A)
in `build_app`. Some `init_app_state` callers (e.g. `run_batch.py`)
build their own bare `State` without going through `build_app`. As a result
`endpoint_plugins` may be absent and are treated that the same as "none attached".
`engine_client` is `None` for the CPU only render server which has no
engine (see `init_render_app_state`). Plugins must handle a `None`
`engine_client` themselves (see `EndpointPlugin.init_state`).
"""
for plugin in getattr(state, "endpoint_plugins", []):
await plugin.init_state(engine_client, state, args)
@asynccontextmanager
async def build_async_engine_client(
args: Namespace,
@@ -230,6 +265,12 @@ def build_app(
register_pooling_api_routers(app, supported_tasks, model_config)
# Endpoint plugins are attached last so their routes are registered after all core
# routers. This runs even for the CPU only render server. A plugin eligible for
# the `render` task still gets its routes registered. It receives
# `engine_client=None` at Phase B (see `_init_endpoint_plugins_state`).
_attach_endpoint_plugins(app, supported_tasks)
app.root_path = args.root_path
app.add_middleware(
CORSMiddleware,
@@ -417,6 +458,8 @@ async def init_app_state(
init_pooling_state(engine_client, state, args, request_logger, supported_tasks)
await _init_endpoint_plugins_state(engine_client, state, args)
state.enable_server_load_tracking = args.enable_server_load_tracking
state.server_load_metrics = 0
@@ -508,6 +551,10 @@ async def init_render_app_state(
state.enable_server_load_tracking = False
state.server_load_metrics = 0
# No `EngineClient` exists for the render server, so plugins get `None` and
# must handle it themselves (see `EndpointPlugin.init_state`).
await _init_endpoint_plugins_state(None, state, args)
def create_server_socket(
addr: tuple[str, int],