[Frontend] Gate scale-out endpoints behind opt-in flag (#54579)

Signed-off-by: Francisco Javier Arceo <farceo@redhat.com>
This commit is contained in:
Francisco Javier Arceo
2026-09-02 14:31:14 +08:00
committed by GitHub
parent 300f688321
commit c00091e026
20 changed files with 386 additions and 39 deletions
+3 -1
View File
@@ -58,6 +58,7 @@ def _enable_cohere_api(monkeypatch):
flag inside the test body.
"""
monkeypatch.setenv("VLLM_ENABLE_COHERE_API", "1")
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "1")
# ----------------------------------------------------------------------
@@ -402,11 +403,12 @@ class TestRenderEndpoint:
assert "/cohere/v2/chat/render" in paths
def test_route_not_registered_when_flag_unset(self, monkeypatch):
monkeypatch.delenv("VLLM_ENABLE_COHERE_API", raising=False)
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
app = FastAPI()
attach_router(app)
paths = [getattr(r, "path", None) for r in app.routes]
assert "/cohere/v2/chat/render" not in paths
assert "/cohere/v2/chat" in paths
def test_returns_generate_request_json(self):
render_handler = _RenderHandler(_generate_request())
+1
View File
@@ -71,6 +71,7 @@ def server():
env_dict = {
"VLLM_CPU_KVCACHE_SPACE": "1",
"VLLM_ENABLE_COHERE_API": "1",
"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1",
}
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server:
@@ -54,7 +54,11 @@ FORCE_WEATHER_TOOL = {"type": "function", "function": {"name": "get_weather"}}
@pytest.fixture(scope="module")
def server():
with RemoteOpenAIServer(MODEL, ARGS) as remote_server:
with RemoteOpenAIServer(
MODEL,
ARGS,
env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"},
) as remote_server:
yield remote_server
@@ -29,7 +29,7 @@ def vision_server():
"0",
]
env_overrides: dict[str, str] = {}
env_overrides = {"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"}
with RemoteOpenAIServer(
VISION_MODEL_NAME,
@@ -0,0 +1,89 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from argparse import Namespace
import pytest
from fastapi import FastAPI
from vllm.entrypoints.launchers.api_server.routers import register_api_routers
from vllm.tasks import SupportedTask
RENDER_PATHS = {
"/v1/chat/completions/render",
"/v1/chat/completions/derender",
"/v1/completions/render",
"/v1/completions/derender",
"/v1/messages/render",
}
GENERATE_PATH = "/inference/v1/generate"
SCALE_OUT_PATHS = RENDER_PATHS | {GENERATE_PATH}
def registered_paths(
supported_tasks: tuple[SupportedTask, ...], *, tokens_only: bool = False
) -> set[str]:
app = FastAPI()
args = Namespace(tokens_only=tokens_only, enable_fault_tolerance=False)
app.state.args = args
register_api_routers(args, app, supported_tasks)
return {route.path for route in app.routes}
def test_scale_out_routes_are_disabled_by_default(monkeypatch):
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
paths = registered_paths(("generate",))
assert SCALE_OUT_PATHS.isdisjoint(paths)
def test_disabled_scale_out_routes_are_logged(monkeypatch, caplog):
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
with caplog.at_level("INFO", logger="vllm.entrypoints.scale_out.factories"):
registered_paths(("generate",))
assert any(
"VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1" in record.message
for record in caplog.records
)
def test_scale_out_routes_can_be_enabled_for_generate_server(monkeypatch):
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "1")
paths = registered_paths(("generate",))
assert paths >= SCALE_OUT_PATHS
def test_render_routes_remain_enabled_for_render_server(monkeypatch):
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
paths = registered_paths(("render",))
assert paths >= RENDER_PATHS
assert GENERATE_PATH not in paths
def test_render_server_rejects_explicitly_disabled_scale_out_routes(monkeypatch):
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "0")
with pytest.raises(ValueError, match="VLLM_ENABLE_SCALE_OUT_ENDPOINTS=0"):
registered_paths(("render",))
def test_tokens_only_mode_enables_generate_routes_when_flag_is_unset(monkeypatch):
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
paths = registered_paths(("generate",), tokens_only=True)
assert paths >= {GENERATE_PATH, "/abort_requests"}
def test_tokens_only_mode_rejects_explicitly_disabled_scale_out_routes(monkeypatch):
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "0")
with pytest.raises(ValueError, match="--tokens-only"):
registered_paths(("generate",), tokens_only=True)
@@ -35,7 +35,11 @@ def server():
"--hf-overrides",
'{"sliding_window": null}',
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
with RemoteOpenAIServer(
MODEL_NAME,
args,
env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"},
) as remote_server:
yield remote_server
@@ -39,7 +39,11 @@ def server():
"--no-enable-prefix-caching",
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
with RemoteOpenAIServer(
MODEL_NAME,
args,
env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"},
) as remote_server:
yield remote_server
@@ -79,7 +79,11 @@ def server(request):
else [str(extra_args)]
)
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
with RemoteOpenAIServer(
MODEL_NAME,
args,
env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"},
) as remote_server:
yield remote_server