[Frontend] Add endpoint plugins framework (#47454)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
@@ -3,10 +3,14 @@
|
||||
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import vllm.envs as envs
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.plugins.endpoint_plugins.interface import EndpointPlugin
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Default plugins group will be loaded in all processes(process0, engine core
|
||||
@@ -20,6 +24,10 @@ PLATFORM_PLUGINS_GROUP = "vllm.platform_plugins"
|
||||
# Stat logger plugins group will be loaded in process0 only when serve vLLM with
|
||||
# async mode.
|
||||
STAT_LOGGER_PLUGINS_GROUP = "vllm.stat_logger_plugins"
|
||||
# Endpoint plugins group is loaded in the API server front end process only.
|
||||
# Each entry point resolves to a factory returning an `EndpointPlugin`
|
||||
# (see `vllm/plugins/endpoint_plugins/interface.py`).
|
||||
ENDPOINT_PLUGINS_GROUP = "vllm.endpoint_plugins"
|
||||
|
||||
# make sure one process only loads plugins once
|
||||
plugins_loaded = False
|
||||
@@ -80,3 +88,71 @@ def load_general_plugins():
|
||||
# general plugins, we only need to execute the loaded functions
|
||||
for func in plugins.values():
|
||||
func()
|
||||
|
||||
|
||||
def load_endpoint_plugins(
|
||||
supported_tasks: "tuple[SupportedTask, ...] | None" = None,
|
||||
) -> "list[EndpointPlugin]":
|
||||
"""Discover, gate and instantiate `vllm.endpoint_plugins` entry points.
|
||||
|
||||
Endpoint plugins add HTTP routes to the API server, so they default to
|
||||
not loading. Unlike other plugin groups, a plugin here is only
|
||||
considered when it is explicitly named in `VLLM_PLUGINS`. This is a
|
||||
stricter posture than `load_plugins_by_group` which "load everything unless
|
||||
an allowlist says otherwise". This posture is taken to handle potentially
|
||||
larger exposed network surface.
|
||||
|
||||
A discovered plugin is loaded only if both hold:
|
||||
- it is named in `VLLM_PLUGINS` (enforced by not calling the loader
|
||||
at all when `VLLM_PLUGINS` is unset). Note that `VLLM_PLUGINS=""`
|
||||
parses to `[""]`, not `None`, so it is treated as a (non strict)
|
||||
allowlist that matches no plugin name, not as "unset".
|
||||
- its `required_tasks` is `None` or intersects `supported_tasks`.
|
||||
|
||||
Args:
|
||||
supported_tasks: Tasks the server supports. `None` means no plugin
|
||||
with a non `None` `required_tasks` will be loaded.
|
||||
|
||||
Returns:
|
||||
Instantiated plugins that passed gating in discovery order.
|
||||
"""
|
||||
from importlib.metadata import entry_points
|
||||
|
||||
if envs.VLLM_PLUGINS is None:
|
||||
discovered = entry_points(group=ENDPOINT_PLUGINS_GROUP)
|
||||
if discovered:
|
||||
logger.warning(
|
||||
"Found endpoint plugin(s) %s but VLLM_PLUGINS is not set. "
|
||||
"Endpoint plugins add HTTP routes and must be explicitly "
|
||||
"allowlisted via VLLM_PLUGINS to be loaded.",
|
||||
[p.name for p in discovered],
|
||||
)
|
||||
return []
|
||||
|
||||
factories = load_plugins_by_group(ENDPOINT_PLUGINS_GROUP)
|
||||
|
||||
endpoint_plugins: list[EndpointPlugin] = []
|
||||
for name, factory in factories.items():
|
||||
try:
|
||||
plugin = factory()
|
||||
except Exception:
|
||||
logger.exception("Failed to instantiate endpoint plugin %s", name)
|
||||
continue
|
||||
|
||||
required_tasks = plugin.required_tasks
|
||||
if required_tasks is not None and (
|
||||
supported_tasks is None or not set(required_tasks) & set(supported_tasks)
|
||||
):
|
||||
logger.info(
|
||||
"Skipping endpoint plugin %s: requires one of tasks %s, "
|
||||
"server supports %s",
|
||||
name,
|
||||
required_tasks,
|
||||
supported_tasks,
|
||||
)
|
||||
continue
|
||||
|
||||
logger.info("Loaded endpoint plugin %s", name)
|
||||
endpoint_plugins.append(plugin)
|
||||
|
||||
return endpoint_plugins
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Contract for `vllm.endpoint_plugins` entry points.
|
||||
|
||||
An endpoint plugin adds HTTP routes to the OpenAI compatible API server.
|
||||
Its scope is HTTP surface only. It registers routes and optionally
|
||||
per app state used by those routes. It must not open new paths into the
|
||||
engine by reaching the engine the same way an in-tree serving handler does
|
||||
via `EngineClient` (e.g. `engine_client.collective_rpc(...)`).
|
||||
|
||||
If a plugin also needs engine side behavior (a new worker side RPC method,
|
||||
a custom stat, etc.) pair this entry point with one registered under
|
||||
`vllm.general_plugins` (see `vllm/plugins/__init__.py`). The
|
||||
`general_plugins` entry installs the engine side method and the
|
||||
`endpoint_plugins` entry exposes it over HTTP. The two are registered and
|
||||
loaded independently where neither implies the other.
|
||||
|
||||
Plugins are opt-in. See `load_endpoint_plugins` in `vllm/plugins/__init__.py`
|
||||
for the loading/gating rules and `docs/usage/security.md` for the security
|
||||
posture of exposing plugin defined routes.
|
||||
|
||||
The CPU only render server (see `build_and_serve_renderer` in
|
||||
`vllm/entrypoints/openai/api_server.py`) has no `EngineClient`. A plugin
|
||||
eligible for the `render` task (`required_tasks` is `None` or includes
|
||||
`"render"`) still gets `attach_router` called but `init_state` receives
|
||||
`engine_client=None`. Plugins that cannot function without an engine should
|
||||
either exclude `"render"` from `required_tasks` or check for `None` in
|
||||
`init_state`/their route handlers and degrade gracefully.
|
||||
"""
|
||||
|
||||
from argparse import Namespace
|
||||
from typing import TYPE_CHECKING, Protocol, runtime_checkable
|
||||
|
||||
from fastapi import FastAPI
|
||||
from starlette.datastructures import State
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class EndpointPlugin(Protocol):
|
||||
"""Protocol implemented by `vllm.endpoint_plugins` entry point factories.
|
||||
|
||||
An entry point registered under the `vllm.endpoint_plugins` group must
|
||||
resolve to a zero argument callable (a class or factory function) that
|
||||
returns an object satisfying this protocol.
|
||||
"""
|
||||
|
||||
name: str
|
||||
"""Unique plugin name used in logs and for `VLLM_PLUGINS` allowlisting."""
|
||||
|
||||
required_tasks: "tuple[SupportedTask, ...] | None"
|
||||
"""Tasks the server must support for this plugin to be loaded.
|
||||
|
||||
The plugin is loaded only if this set intersects the server's
|
||||
`supported_tasks`. `None` means the plugin has no task requirement and
|
||||
is always eligible (subject to the `VLLM_PLUGINS` allowlist).
|
||||
"""
|
||||
|
||||
def attach_router(self, app: FastAPI) -> None:
|
||||
"""Register this plugin's routes on `app`.
|
||||
|
||||
Called once during `build_app()` after all core routers have been
|
||||
attached. Routes attached here can shadow core routes with the same
|
||||
path. There is currently no conflict enforcement (see RFC #46565 follow ups).
|
||||
"""
|
||||
...
|
||||
|
||||
async def init_state(
|
||||
self, engine_client: "EngineClient | None", state: State, args: Namespace
|
||||
) -> None:
|
||||
"""Initialize per app state consumed by this plugin's routes.
|
||||
|
||||
Called once during `init_app_state()` after core state has been
|
||||
initialized. Use `engine_client` (e.g. `collective_rpc`) to reach
|
||||
the engine. Do not open new engine access paths.
|
||||
|
||||
`engine_client` is `None` on the CPU only render server which has
|
||||
no engine. This only happens for plugins eligible for the `render`
|
||||
task (`required_tasks` is `None` or includes `"render"`). Handle
|
||||
`None` explicitly (e.g. skip engine dependent setup, or have route
|
||||
handlers return an error) if the plugin is loadable for `render` but
|
||||
cannot function without an engine.
|
||||
"""
|
||||
...
|
||||
Reference in New Issue
Block a user