[Frontend] Replace VLLM_ENABLE_SCALE_OUT_ENDPOINTS with --enable-scale-out (#55176)

Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
Martin Hickey
2026-09-14 10:02:14 +00:00
committed by GitHub
parent 2c7ee87223
commit 934b1fcbcb
24 changed files with 130 additions and 215 deletions
@@ -34,6 +34,7 @@ ARGS = [
"hermes",
"--reasoning-parser",
"deepseek_r1",
"--enable-scale-out",
]
TOOLS = [
@@ -54,11 +55,7 @@ FORCE_WEATHER_TOOL = {"type": "function", "function": {"name": "get_weather"}}
@pytest.fixture(scope="module")
def server():
with RemoteOpenAIServer(
MODEL,
ARGS,
env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"},
) as remote_server:
with RemoteOpenAIServer(MODEL, ARGS) as remote_server:
yield remote_server
@@ -27,15 +27,10 @@ def vision_server():
"1",
"--limit-mm-per-prompt.video",
"0",
"--enable-scale-out",
]
env_overrides = {"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"}
with RemoteOpenAIServer(
VISION_MODEL_NAME,
args,
env_dict=env_overrides,
) as remote_server:
with RemoteOpenAIServer(VISION_MODEL_NAME, args) as remote_server:
yield remote_server
+16 -37
View File
@@ -3,7 +3,6 @@
from argparse import Namespace
import pytest
from fastapi import FastAPI
from vllm.entrypoints.launchers.api_server.routers import register_api_routers
@@ -22,69 +21,49 @@ SCALE_OUT_PATHS = RENDER_PATHS | {GENERATE_PATH}
def registered_paths(
supported_tasks: tuple[SupportedTask, ...], *, tokens_only: bool = False
supported_tasks: tuple[SupportedTask, ...],
*,
tokens_only: bool = False,
enable_scale_out: bool = False,
) -> set[str]:
app = FastAPI()
args = Namespace(tokens_only=tokens_only, enable_fault_tolerance=False)
args = Namespace(
tokens_only=tokens_only,
enable_scale_out=enable_scale_out,
enable_fault_tolerance=False,
)
app.state.args = args
register_api_routers(args, app, supported_tasks)
return {route.path for route in app.routes}
def test_scale_out_routes_are_disabled_by_default(monkeypatch):
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
def test_scale_out_routes_are_disabled_by_default():
paths = registered_paths(("generate",))
assert SCALE_OUT_PATHS.isdisjoint(paths)
def test_disabled_scale_out_routes_are_logged(monkeypatch, caplog):
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
def test_disabled_scale_out_routes_are_logged(caplog):
with caplog.at_level("INFO", logger="vllm.entrypoints.scale_out.factories"):
registered_paths(("generate",))
assert any(
"VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1" in record.message
for record in caplog.records
)
assert any("--enable-scale-out" in record.message for record in caplog.records)
def test_scale_out_routes_can_be_enabled_for_generate_server(monkeypatch):
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "1")
paths = registered_paths(("generate",))
def test_scale_out_routes_can_be_enabled_for_generate_server():
paths = registered_paths(("generate",), enable_scale_out=True)
assert paths >= SCALE_OUT_PATHS
def test_render_routes_remain_enabled_for_render_server(monkeypatch):
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
def test_render_routes_remain_enabled_for_render_server():
paths = registered_paths(("render",))
assert paths >= RENDER_PATHS
assert GENERATE_PATH not in paths
def test_render_server_rejects_explicitly_disabled_scale_out_routes(monkeypatch):
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "0")
with pytest.raises(ValueError, match="VLLM_ENABLE_SCALE_OUT_ENDPOINTS=0"):
registered_paths(("render",))
def test_tokens_only_mode_enables_generate_routes_when_flag_is_unset(monkeypatch):
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
def test_tokens_only_mode_enables_generate_routes_when_flag_is_unset():
paths = registered_paths(("generate",), tokens_only=True)
assert paths >= {GENERATE_PATH, "/abort_requests"}
def test_tokens_only_mode_rejects_explicitly_disabled_scale_out_routes(monkeypatch):
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "0")
with pytest.raises(ValueError, match="--tokens-only"):
registered_paths(("generate",), tokens_only=True)
@@ -34,12 +34,9 @@ def server():
"--enable-return-routed-experts",
"--hf-overrides",
'{"sliding_window": null}',
"--enable-scale-out",
]
with RemoteOpenAIServer(
MODEL_NAME,
args,
env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"},
) as remote_server:
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@@ -37,13 +37,10 @@ def server():
"4096",
"--enforce-eager",
"--no-enable-prefix-caching",
"--enable-scale-out",
]
with RemoteOpenAIServer(
MODEL_NAME,
args,
env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"},
) as remote_server:
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@@ -69,6 +69,7 @@ def server(request):
# Option B: deterministic prefix caching
# "--enable-prefix-caching",
# "--deterministic-prefix-caching",
"--enable-scale-out",
]
extra_args = getattr(request, "param", None)
@@ -79,11 +80,7 @@ def server(request):
else [str(extra_args)]
)
with RemoteOpenAIServer(
MODEL_NAME,
args,
env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"},
) as remote_server:
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server