[Frontend] Add endpoint plugins framework (#47454)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
@@ -74,6 +74,41 @@ logger = init_logger("vllm.entrypoints.openai.api_server")
|
||||
_FALLBACK_SUPPORTED_TASKS: tuple[SupportedTask, ...] = ("generate",)
|
||||
|
||||
|
||||
def _attach_endpoint_plugins(
|
||||
app: FastAPI, supported_tasks: tuple["SupportedTask", ...]
|
||||
) -> None:
|
||||
"""Phase A of endpoint plugin wiring: discover, gate and attach routes.
|
||||
|
||||
Attached last after all core routers. This is so endpoint plugin routes can
|
||||
shadow core routes with the same path (see `EndpointPlugin.attach_router`
|
||||
docstring). No-ops when no plugins are discovered/allowlisted.
|
||||
"""
|
||||
from vllm.plugins import load_endpoint_plugins
|
||||
|
||||
endpoint_plugins = load_endpoint_plugins(supported_tasks)
|
||||
for plugin in endpoint_plugins:
|
||||
plugin.attach_router(app)
|
||||
app.state.endpoint_plugins = endpoint_plugins
|
||||
|
||||
|
||||
async def _init_endpoint_plugins_state(
|
||||
engine_client: EngineClient | None, state: State, args: Namespace
|
||||
) -> None:
|
||||
"""Phase B of endpoint plugin wiring: initialize per app plugin state.
|
||||
|
||||
`state.endpoint_plugins` is set by `_attach_endpoint_plugins` (Phase A)
|
||||
in `build_app`. Some `init_app_state` callers (e.g. `run_batch.py`)
|
||||
build their own bare `State` without going through `build_app`. As a result
|
||||
`endpoint_plugins` may be absent and are treated that the same as "none attached".
|
||||
|
||||
`engine_client` is `None` for the CPU only render server which has no
|
||||
engine (see `init_render_app_state`). Plugins must handle a `None`
|
||||
`engine_client` themselves (see `EndpointPlugin.init_state`).
|
||||
"""
|
||||
for plugin in getattr(state, "endpoint_plugins", []):
|
||||
await plugin.init_state(engine_client, state, args)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def build_async_engine_client(
|
||||
args: Namespace,
|
||||
@@ -230,6 +265,12 @@ def build_app(
|
||||
|
||||
register_pooling_api_routers(app, supported_tasks, model_config)
|
||||
|
||||
# Endpoint plugins are attached last so their routes are registered after all core
|
||||
# routers. This runs even for the CPU only render server. A plugin eligible for
|
||||
# the `render` task still gets its routes registered. It receives
|
||||
# `engine_client=None` at Phase B (see `_init_endpoint_plugins_state`).
|
||||
_attach_endpoint_plugins(app, supported_tasks)
|
||||
|
||||
app.root_path = args.root_path
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
@@ -417,6 +458,8 @@ async def init_app_state(
|
||||
|
||||
init_pooling_state(engine_client, state, args, request_logger, supported_tasks)
|
||||
|
||||
await _init_endpoint_plugins_state(engine_client, state, args)
|
||||
|
||||
state.enable_server_load_tracking = args.enable_server_load_tracking
|
||||
state.server_load_metrics = 0
|
||||
|
||||
@@ -508,6 +551,10 @@ async def init_render_app_state(
|
||||
state.enable_server_load_tracking = False
|
||||
state.server_load_metrics = 0
|
||||
|
||||
# No `EngineClient` exists for the render server, so plugins get `None` and
|
||||
# must handle it themselves (see `EndpointPlugin.init_state`).
|
||||
await _init_endpoint_plugins_state(None, state, args)
|
||||
|
||||
|
||||
def create_server_socket(
|
||||
addr: tuple[str, int],
|
||||
|
||||
Reference in New Issue
Block a user