[Frontend] Add endpoint plugins framework (#47454)

Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
Martin Hickey
2026-07-08 10:00:41 +08:00
committed by GitHub
parent f7efab58ec
commit f7fc0ca993
12 changed files with 660 additions and 1 deletions
+77 -1
View File
@@ -3,10 +3,14 @@
import logging
from collections.abc import Callable
from typing import Any
from typing import TYPE_CHECKING, Any
import vllm.envs as envs
if TYPE_CHECKING:
from vllm.plugins.endpoint_plugins.interface import EndpointPlugin
from vllm.tasks import SupportedTask
logger = logging.getLogger(__name__)
# Default plugins group will be loaded in all processes(process0, engine core
@@ -20,6 +24,10 @@ PLATFORM_PLUGINS_GROUP = "vllm.platform_plugins"
# Stat logger plugins group will be loaded in process0 only when serve vLLM with
# async mode.
STAT_LOGGER_PLUGINS_GROUP = "vllm.stat_logger_plugins"
# Endpoint plugins group is loaded in the API server front end process only.
# Each entry point resolves to a factory returning an `EndpointPlugin`
# (see `vllm/plugins/endpoint_plugins/interface.py`).
ENDPOINT_PLUGINS_GROUP = "vllm.endpoint_plugins"
# make sure one process only loads plugins once
plugins_loaded = False
@@ -80,3 +88,71 @@ def load_general_plugins():
# general plugins, we only need to execute the loaded functions
for func in plugins.values():
func()
def load_endpoint_plugins(
supported_tasks: "tuple[SupportedTask, ...] | None" = None,
) -> "list[EndpointPlugin]":
"""Discover, gate and instantiate `vllm.endpoint_plugins` entry points.
Endpoint plugins add HTTP routes to the API server, so they default to
not loading. Unlike other plugin groups, a plugin here is only
considered when it is explicitly named in `VLLM_PLUGINS`. This is a
stricter posture than `load_plugins_by_group` which "load everything unless
an allowlist says otherwise". This posture is taken to handle potentially
larger exposed network surface.
A discovered plugin is loaded only if both hold:
- it is named in `VLLM_PLUGINS` (enforced by not calling the loader
at all when `VLLM_PLUGINS` is unset). Note that `VLLM_PLUGINS=""`
parses to `[""]`, not `None`, so it is treated as a (non strict)
allowlist that matches no plugin name, not as "unset".
- its `required_tasks` is `None` or intersects `supported_tasks`.
Args:
supported_tasks: Tasks the server supports. `None` means no plugin
with a non `None` `required_tasks` will be loaded.
Returns:
Instantiated plugins that passed gating in discovery order.
"""
from importlib.metadata import entry_points
if envs.VLLM_PLUGINS is None:
discovered = entry_points(group=ENDPOINT_PLUGINS_GROUP)
if discovered:
logger.warning(
"Found endpoint plugin(s) %s but VLLM_PLUGINS is not set. "
"Endpoint plugins add HTTP routes and must be explicitly "
"allowlisted via VLLM_PLUGINS to be loaded.",
[p.name for p in discovered],
)
return []
factories = load_plugins_by_group(ENDPOINT_PLUGINS_GROUP)
endpoint_plugins: list[EndpointPlugin] = []
for name, factory in factories.items():
try:
plugin = factory()
except Exception:
logger.exception("Failed to instantiate endpoint plugin %s", name)
continue
required_tasks = plugin.required_tasks
if required_tasks is not None and (
supported_tasks is None or not set(required_tasks) & set(supported_tasks)
):
logger.info(
"Skipping endpoint plugin %s: requires one of tasks %s, "
"server supports %s",
name,
required_tasks,
supported_tasks,
)
continue
logger.info("Loaded endpoint plugin %s", name)
endpoint_plugins.append(plugin)
return endpoint_plugins
@@ -0,0 +1,87 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Contract for `vllm.endpoint_plugins` entry points.
An endpoint plugin adds HTTP routes to the OpenAI compatible API server.
Its scope is HTTP surface only. It registers routes and optionally
per app state used by those routes. It must not open new paths into the
engine by reaching the engine the same way an in-tree serving handler does
via `EngineClient` (e.g. `engine_client.collective_rpc(...)`).
If a plugin also needs engine side behavior (a new worker side RPC method,
a custom stat, etc.) pair this entry point with one registered under
`vllm.general_plugins` (see `vllm/plugins/__init__.py`). The
`general_plugins` entry installs the engine side method and the
`endpoint_plugins` entry exposes it over HTTP. The two are registered and
loaded independently where neither implies the other.
Plugins are opt-in. See `load_endpoint_plugins` in `vllm/plugins/__init__.py`
for the loading/gating rules and `docs/usage/security.md` for the security
posture of exposing plugin defined routes.
The CPU only render server (see `build_and_serve_renderer` in
`vllm/entrypoints/openai/api_server.py`) has no `EngineClient`. A plugin
eligible for the `render` task (`required_tasks` is `None` or includes
`"render"`) still gets `attach_router` called but `init_state` receives
`engine_client=None`. Plugins that cannot function without an engine should
either exclude `"render"` from `required_tasks` or check for `None` in
`init_state`/their route handlers and degrade gracefully.
"""
from argparse import Namespace
from typing import TYPE_CHECKING, Protocol, runtime_checkable
from fastapi import FastAPI
from starlette.datastructures import State
if TYPE_CHECKING:
from vllm.engine.protocol import EngineClient
from vllm.tasks import SupportedTask
@runtime_checkable
class EndpointPlugin(Protocol):
"""Protocol implemented by `vllm.endpoint_plugins` entry point factories.
An entry point registered under the `vllm.endpoint_plugins` group must
resolve to a zero argument callable (a class or factory function) that
returns an object satisfying this protocol.
"""
name: str
"""Unique plugin name used in logs and for `VLLM_PLUGINS` allowlisting."""
required_tasks: "tuple[SupportedTask, ...] | None"
"""Tasks the server must support for this plugin to be loaded.
The plugin is loaded only if this set intersects the server's
`supported_tasks`. `None` means the plugin has no task requirement and
is always eligible (subject to the `VLLM_PLUGINS` allowlist).
"""
def attach_router(self, app: FastAPI) -> None:
"""Register this plugin's routes on `app`.
Called once during `build_app()` after all core routers have been
attached. Routes attached here can shadow core routes with the same
path. There is currently no conflict enforcement (see RFC #46565 follow ups).
"""
...
async def init_state(
self, engine_client: "EngineClient | None", state: State, args: Namespace
) -> None:
"""Initialize per app state consumed by this plugin's routes.
Called once during `init_app_state()` after core state has been
initialized. Use `engine_client` (e.g. `collective_rpc`) to reach
the engine. Do not open new engine access paths.
`engine_client` is `None` on the CPU only render server which has
no engine. This only happens for plugins eligible for the `render`
task (`required_tasks` is `None` or includes `"render"`). Handle
`None` explicitly (e.g. skip engine dependent setup, or have route
handlers return an error) if the plugin is loadable for `render` but
cannot function without an engine.
"""
...