[Frontend] Add endpoint plugins framework (#47454)

Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
Martin Hickey
2026-07-08 10:00:41 +08:00
committed by GitHub
parent f7efab58ec
commit f7fc0ca993
12 changed files with 660 additions and 1 deletions
+77 -1
View File
@@ -3,10 +3,14 @@
import logging
from collections.abc import Callable
from typing import Any
from typing import TYPE_CHECKING, Any
import vllm.envs as envs
if TYPE_CHECKING:
from vllm.plugins.endpoint_plugins.interface import EndpointPlugin
from vllm.tasks import SupportedTask
logger = logging.getLogger(__name__)
# Default plugins group will be loaded in all processes(process0, engine core
@@ -20,6 +24,10 @@ PLATFORM_PLUGINS_GROUP = "vllm.platform_plugins"
# Stat logger plugins group will be loaded in process0 only when serve vLLM with
# async mode.
STAT_LOGGER_PLUGINS_GROUP = "vllm.stat_logger_plugins"
# Endpoint plugins group is loaded in the API server front end process only.
# Each entry point resolves to a factory returning an `EndpointPlugin`
# (see `vllm/plugins/endpoint_plugins/interface.py`).
ENDPOINT_PLUGINS_GROUP = "vllm.endpoint_plugins"
# make sure one process only loads plugins once
plugins_loaded = False
@@ -80,3 +88,71 @@ def load_general_plugins():
# general plugins, we only need to execute the loaded functions
for func in plugins.values():
func()
def load_endpoint_plugins(
supported_tasks: "tuple[SupportedTask, ...] | None" = None,
) -> "list[EndpointPlugin]":
"""Discover, gate and instantiate `vllm.endpoint_plugins` entry points.
Endpoint plugins add HTTP routes to the API server, so they default to
not loading. Unlike other plugin groups, a plugin here is only
considered when it is explicitly named in `VLLM_PLUGINS`. This is a
stricter posture than `load_plugins_by_group` which "load everything unless
an allowlist says otherwise". This posture is taken to handle potentially
larger exposed network surface.
A discovered plugin is loaded only if both hold:
- it is named in `VLLM_PLUGINS` (enforced by not calling the loader
at all when `VLLM_PLUGINS` is unset). Note that `VLLM_PLUGINS=""`
parses to `[""]`, not `None`, so it is treated as a (non strict)
allowlist that matches no plugin name, not as "unset".
- its `required_tasks` is `None` or intersects `supported_tasks`.
Args:
supported_tasks: Tasks the server supports. `None` means no plugin
with a non `None` `required_tasks` will be loaded.
Returns:
Instantiated plugins that passed gating in discovery order.
"""
from importlib.metadata import entry_points
if envs.VLLM_PLUGINS is None:
discovered = entry_points(group=ENDPOINT_PLUGINS_GROUP)
if discovered:
logger.warning(
"Found endpoint plugin(s) %s but VLLM_PLUGINS is not set. "
"Endpoint plugins add HTTP routes and must be explicitly "
"allowlisted via VLLM_PLUGINS to be loaded.",
[p.name for p in discovered],
)
return []
factories = load_plugins_by_group(ENDPOINT_PLUGINS_GROUP)
endpoint_plugins: list[EndpointPlugin] = []
for name, factory in factories.items():
try:
plugin = factory()
except Exception:
logger.exception("Failed to instantiate endpoint plugin %s", name)
continue
required_tasks = plugin.required_tasks
if required_tasks is not None and (
supported_tasks is None or not set(required_tasks) & set(supported_tasks)
):
logger.info(
"Skipping endpoint plugin %s: requires one of tasks %s, "
"server supports %s",
name,
required_tasks,
supported_tasks,
)
continue
logger.info("Loaded endpoint plugin %s", name)
endpoint_plugins.append(plugin)
return endpoint_plugins