[Frontend] Replace VLLM_ENABLE_SCALE_OUT_ENDPOINTS with --enable-scale-out (#55176)

Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
Martin Hickey
2026-09-14 10:02:14 +00:00
committed by GitHub
parent 2c7ee87223
commit 934b1fcbcb
24 changed files with 130 additions and 215 deletions
+6 -4
View File
@@ -17,9 +17,10 @@ log) and vLLM continues to boot normally.
Even when the SDK is installed, :func:`attach_router` also requires
``VLLM_ENABLE_COHERE_API=1`` in the environment before it will expose
the chat route. The render route additionally requires
``VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1``. These gates keep unrelated
deployments from accidentally exposing the APIs.
the chat route. The render route additionally requires scale-out
endpoints to be enabled, i.e. ``--enable-scale-out`` or
``--tokens-only``. These gates keep unrelated deployments from
accidentally exposing the APIs.
Note: the handlers must live at module scope (not inside
``attach_router``) so that FastAPI's ``typing.get_type_hints`` resolves
@@ -306,6 +307,7 @@ def attach_router(app: FastAPI) -> None:
)
return
app.include_router(router)
if envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS:
args = getattr(app.state, "args", None)
if getattr(args, "enable_scale_out", False) or getattr(args, "tokens_only", False):
app.include_router(render_router)
app.add_middleware(CohereErrorEnvelopeMiddleware)
+7
View File
@@ -181,6 +181,13 @@ class BaseFrontendArgs:
If set to True, only enable the Tokens In<>Out endpoint.
This is intended for use in a Disaggregated Everything setup.
"""
enable_scale_out: bool = False
"""
If set to True, register the scale-out endpoints (`/render`, `/derender`,
and `/inference/v1/generate`) on `vllm serve`. Has no effect on
`vllm launch render` or `vllm serve --tokens-only`, which always register
their required endpoints regardless of this flag.
"""
fingerprint_mode: Literal["full", "hash", "custom", "none"] = "full"
"""Controls the ``system_fingerprint`` field on responses.
+8 -16
View File
@@ -5,7 +5,6 @@ from typing import TYPE_CHECKING
from fastapi import FastAPI
from vllm import envs
from vllm.engine.protocol import EngineClient
from vllm.logger import init_logger
from vllm.tasks import SupportedTask
@@ -67,22 +66,15 @@ def register_scale_out_api_routers(
app: FastAPI,
supported_tasks: tuple["SupportedTask", ...],
):
dedicated_mode = None
if "render" in supported_tasks:
dedicated_mode = "`vllm launch render`"
elif getattr(app.state.args, "tokens_only", False):
dedicated_mode = "`--tokens-only`"
enabled = envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS
if dedicated_mode is not None and enabled is False:
raise ValueError(
"`VLLM_ENABLE_SCALE_OUT_ENDPOINTS=0` conflicts with "
f"{dedicated_mode}; unset it or set it to 1"
)
if dedicated_mode is None and not enabled:
args = getattr(app.state, "args", None)
# `vllm launch render` and `vllm serve --tokens-only` are dedicated to
# serving these endpoints, so they always register them regardless of
# `--enable-scale-out`.
dedicated_mode = "render" in supported_tasks or getattr(args, "tokens_only", False)
enabled = dedicated_mode or getattr(args, "enable_scale_out", False)
if not enabled:
logger.info(
"Scale-out endpoints are disabled. Set "
"VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 to enable them."
"Scale-out endpoints are disabled. Set --enable-scale-out to enable them."
)
return
-17
View File
@@ -261,7 +261,6 @@ if TYPE_CHECKING:
VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE: bool = True
VLLM_ENABLE_RESPONSES_API_STORE: bool = False
VLLM_ENABLE_COHERE_API: bool = False
VLLM_ENABLE_SCALE_OUT_ENDPOINTS: bool | None = None
VLLM_HAS_FLASHINFER_CUBIN: bool = False
VLLM_ROCM_FP8_MFMA_PAGE_ATTN: bool = False
VLLM_ALLREDUCE_USE_SYMM_MEM: bool = True
@@ -352,15 +351,6 @@ def maybe_convert_bool(value: str | None) -> bool | None:
return bool(int(value))
def maybe_convert_scale_out_endpoints(value: str | None) -> bool | None:
if value is None:
return None
normalized = value.strip()
if normalized not in ("0", "1"):
raise ValueError("VLLM_ENABLE_SCALE_OUT_ENDPOINTS must be 0 or 1")
return maybe_convert_bool(normalized)
def maybe_convert_json_str_or_file(value: str | None) -> dict[str, Any] | None:
if value is None:
return None
@@ -1858,12 +1848,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_ENABLE_COHERE_API": lambda: bool(
int(os.getenv("VLLM_ENABLE_COHERE_API", "0"))
),
# If set to 1, expose the scale-out endpoints on `vllm serve`.
# The dedicated `vllm launch render` server exposes render and derender
# endpoints when this variable is unset or set to 1.
"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": lambda: maybe_convert_scale_out_endpoints(
os.getenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS") or None
),
# If set, use the fp8 mfma in rocm paged attention.
"VLLM_ROCM_FP8_MFMA_PAGE_ATTN": lambda: bool(
int(os.getenv("VLLM_ROCM_FP8_MFMA_PAGE_ATTN", "0"))
@@ -2289,7 +2273,6 @@ def compile_factors() -> dict[str, object]:
"VLLM_CONFIG_ROOT",
"LD_LIBRARY_PATH",
"VLLM_SERVER_DEV_MODE",
"VLLM_ENABLE_SCALE_OUT_ENDPOINTS",
"VLLM_DP_MASTER_IP",
"VLLM_DP_MASTER_PORT",
"VLLM_NIXL_SIDE_CHANNEL_HOST",