[Frontend] Replace VLLM_ENABLE_SCALE_OUT_ENDPOINTS with --enable-scale-out (#55176)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
@@ -17,9 +17,10 @@ log) and vLLM continues to boot normally.
|
||||
|
||||
Even when the SDK is installed, :func:`attach_router` also requires
|
||||
``VLLM_ENABLE_COHERE_API=1`` in the environment before it will expose
|
||||
the chat route. The render route additionally requires
|
||||
``VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1``. These gates keep unrelated
|
||||
deployments from accidentally exposing the APIs.
|
||||
the chat route. The render route additionally requires scale-out
|
||||
endpoints to be enabled, i.e. ``--enable-scale-out`` or
|
||||
``--tokens-only``. These gates keep unrelated deployments from
|
||||
accidentally exposing the APIs.
|
||||
|
||||
Note: the handlers must live at module scope (not inside
|
||||
``attach_router``) so that FastAPI's ``typing.get_type_hints`` resolves
|
||||
@@ -306,6 +307,7 @@ def attach_router(app: FastAPI) -> None:
|
||||
)
|
||||
return
|
||||
app.include_router(router)
|
||||
if envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS:
|
||||
args = getattr(app.state, "args", None)
|
||||
if getattr(args, "enable_scale_out", False) or getattr(args, "tokens_only", False):
|
||||
app.include_router(render_router)
|
||||
app.add_middleware(CohereErrorEnvelopeMiddleware)
|
||||
|
||||
@@ -181,6 +181,13 @@ class BaseFrontendArgs:
|
||||
If set to True, only enable the Tokens In<>Out endpoint.
|
||||
This is intended for use in a Disaggregated Everything setup.
|
||||
"""
|
||||
enable_scale_out: bool = False
|
||||
"""
|
||||
If set to True, register the scale-out endpoints (`/render`, `/derender`,
|
||||
and `/inference/v1/generate`) on `vllm serve`. Has no effect on
|
||||
`vllm launch render` or `vllm serve --tokens-only`, which always register
|
||||
their required endpoints regardless of this flag.
|
||||
"""
|
||||
fingerprint_mode: Literal["full", "hash", "custom", "none"] = "full"
|
||||
"""Controls the ``system_fingerprint`` field on responses.
|
||||
|
||||
|
||||
@@ -5,7 +5,6 @@ from typing import TYPE_CHECKING
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
from vllm import envs
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tasks import SupportedTask
|
||||
@@ -67,22 +66,15 @@ def register_scale_out_api_routers(
|
||||
app: FastAPI,
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
):
|
||||
dedicated_mode = None
|
||||
if "render" in supported_tasks:
|
||||
dedicated_mode = "`vllm launch render`"
|
||||
elif getattr(app.state.args, "tokens_only", False):
|
||||
dedicated_mode = "`--tokens-only`"
|
||||
|
||||
enabled = envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS
|
||||
if dedicated_mode is not None and enabled is False:
|
||||
raise ValueError(
|
||||
"`VLLM_ENABLE_SCALE_OUT_ENDPOINTS=0` conflicts with "
|
||||
f"{dedicated_mode}; unset it or set it to 1"
|
||||
)
|
||||
if dedicated_mode is None and not enabled:
|
||||
args = getattr(app.state, "args", None)
|
||||
# `vllm launch render` and `vllm serve --tokens-only` are dedicated to
|
||||
# serving these endpoints, so they always register them regardless of
|
||||
# `--enable-scale-out`.
|
||||
dedicated_mode = "render" in supported_tasks or getattr(args, "tokens_only", False)
|
||||
enabled = dedicated_mode or getattr(args, "enable_scale_out", False)
|
||||
if not enabled:
|
||||
logger.info(
|
||||
"Scale-out endpoints are disabled. Set "
|
||||
"VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 to enable them."
|
||||
"Scale-out endpoints are disabled. Set --enable-scale-out to enable them."
|
||||
)
|
||||
return
|
||||
|
||||
|
||||
Reference in New Issue
Block a user