[Frontend] Replace VLLM_ENABLE_SCALE_OUT_ENDPOINTS with --enable-scale-out (#55176)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
@@ -58,7 +58,6 @@ def _enable_cohere_api(monkeypatch):
|
||||
flag inside the test body.
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_ENABLE_COHERE_API", "1")
|
||||
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "1")
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
@@ -128,15 +127,19 @@ def _generate_request() -> GenerateRequest:
|
||||
)
|
||||
|
||||
|
||||
def _build_app(handler: _Handler | None) -> FastAPI:
|
||||
def _build_app(handler: _Handler | None, *, enable_scale_out: bool = True) -> FastAPI:
|
||||
app = FastAPI()
|
||||
app.state.args = Namespace(enable_scale_out=enable_scale_out, tokens_only=False)
|
||||
attach_router(app)
|
||||
app.state.cohere_serving_chat_v2 = handler
|
||||
return app
|
||||
|
||||
|
||||
def _build_render_app(chat_handler, render_handler) -> FastAPI:
|
||||
def _build_render_app(
|
||||
chat_handler, render_handler, *, enable_scale_out: bool = True
|
||||
) -> FastAPI:
|
||||
app = FastAPI()
|
||||
app.state.args = Namespace(enable_scale_out=enable_scale_out, tokens_only=False)
|
||||
attach_router(app)
|
||||
app.state.cohere_serving_chat_v2 = chat_handler
|
||||
app.state.serving_render = render_handler
|
||||
@@ -151,11 +154,13 @@ def _build_app_with_vllm_handlers(handler: _Handler | None) -> FastAPI:
|
||||
into the ``CohereError`` wire shape.
|
||||
"""
|
||||
app = FastAPI()
|
||||
attach_router(app)
|
||||
app.state.cohere_serving_chat_v2 = handler
|
||||
# ``validation_exception_handler`` reads ``req.app.state.args``; the
|
||||
# real cli builds this via argparse.
|
||||
app.state.args = Namespace(log_error_stack=False)
|
||||
app.state.args = Namespace(
|
||||
log_error_stack=False, enable_scale_out=True, tokens_only=False
|
||||
)
|
||||
attach_router(app)
|
||||
app.state.cohere_serving_chat_v2 = handler
|
||||
app.exception_handler(RequestValidationError)(validation_exception_handler)
|
||||
app.exception_handler(HTTPException)(http_exception_handler)
|
||||
app.exception_handler(VLLMError)(vllm_error_handler)
|
||||
@@ -402,10 +407,8 @@ class TestRenderEndpoint:
|
||||
paths = [getattr(r, "path", None) for r in app.routes]
|
||||
assert "/cohere/v2/chat/render" in paths
|
||||
|
||||
def test_route_not_registered_when_flag_unset(self, monkeypatch):
|
||||
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
|
||||
app = FastAPI()
|
||||
attach_router(app)
|
||||
def test_route_not_registered_when_flag_unset(self):
|
||||
app = _build_app(handler=None, enable_scale_out=False)
|
||||
paths = [getattr(r, "path", None) for r in app.routes]
|
||||
assert "/cohere/v2/chat/render" not in paths
|
||||
assert "/cohere/v2/chat" in paths
|
||||
|
||||
@@ -57,6 +57,8 @@ def server():
|
||||
# the conversation to surface a thinking block (SmolLM2 doesn't
|
||||
# emit Cohere-style reasoning tokens out of the box).
|
||||
"--no-cohere-is-reasoning-model",
|
||||
# Registers /cohere/v2/chat/render alongside /cohere/v2/chat.
|
||||
"--enable-scale-out",
|
||||
]
|
||||
|
||||
# Cap the CPU KV-cache pool the vLLM CPU backend reserves at
|
||||
@@ -71,7 +73,6 @@ def server():
|
||||
env_dict = {
|
||||
"VLLM_CPU_KVCACHE_SPACE": "1",
|
||||
"VLLM_ENABLE_COHERE_API": "1",
|
||||
"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1",
|
||||
}
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server:
|
||||
|
||||
@@ -34,6 +34,7 @@ ARGS = [
|
||||
"hermes",
|
||||
"--reasoning-parser",
|
||||
"deepseek_r1",
|
||||
"--enable-scale-out",
|
||||
]
|
||||
|
||||
TOOLS = [
|
||||
@@ -54,11 +55,7 @@ FORCE_WEATHER_TOOL = {"type": "function", "function": {"name": "get_weather"}}
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
with RemoteOpenAIServer(
|
||||
MODEL,
|
||||
ARGS,
|
||||
env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"},
|
||||
) as remote_server:
|
||||
with RemoteOpenAIServer(MODEL, ARGS) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
|
||||
@@ -27,15 +27,10 @@ def vision_server():
|
||||
"1",
|
||||
"--limit-mm-per-prompt.video",
|
||||
"0",
|
||||
"--enable-scale-out",
|
||||
]
|
||||
|
||||
env_overrides = {"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"}
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
VISION_MODEL_NAME,
|
||||
args,
|
||||
env_dict=env_overrides,
|
||||
) as remote_server:
|
||||
with RemoteOpenAIServer(VISION_MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
|
||||
from argparse import Namespace
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
|
||||
from vllm.entrypoints.launchers.api_server.routers import register_api_routers
|
||||
@@ -22,69 +21,49 @@ SCALE_OUT_PATHS = RENDER_PATHS | {GENERATE_PATH}
|
||||
|
||||
|
||||
def registered_paths(
|
||||
supported_tasks: tuple[SupportedTask, ...], *, tokens_only: bool = False
|
||||
supported_tasks: tuple[SupportedTask, ...],
|
||||
*,
|
||||
tokens_only: bool = False,
|
||||
enable_scale_out: bool = False,
|
||||
) -> set[str]:
|
||||
app = FastAPI()
|
||||
args = Namespace(tokens_only=tokens_only, enable_fault_tolerance=False)
|
||||
args = Namespace(
|
||||
tokens_only=tokens_only,
|
||||
enable_scale_out=enable_scale_out,
|
||||
enable_fault_tolerance=False,
|
||||
)
|
||||
app.state.args = args
|
||||
register_api_routers(args, app, supported_tasks)
|
||||
return {route.path for route in app.routes}
|
||||
|
||||
|
||||
def test_scale_out_routes_are_disabled_by_default(monkeypatch):
|
||||
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
|
||||
|
||||
def test_scale_out_routes_are_disabled_by_default():
|
||||
paths = registered_paths(("generate",))
|
||||
|
||||
assert SCALE_OUT_PATHS.isdisjoint(paths)
|
||||
|
||||
|
||||
def test_disabled_scale_out_routes_are_logged(monkeypatch, caplog):
|
||||
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
|
||||
|
||||
def test_disabled_scale_out_routes_are_logged(caplog):
|
||||
with caplog.at_level("INFO", logger="vllm.entrypoints.scale_out.factories"):
|
||||
registered_paths(("generate",))
|
||||
|
||||
assert any(
|
||||
"VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1" in record.message
|
||||
for record in caplog.records
|
||||
)
|
||||
assert any("--enable-scale-out" in record.message for record in caplog.records)
|
||||
|
||||
|
||||
def test_scale_out_routes_can_be_enabled_for_generate_server(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "1")
|
||||
|
||||
paths = registered_paths(("generate",))
|
||||
def test_scale_out_routes_can_be_enabled_for_generate_server():
|
||||
paths = registered_paths(("generate",), enable_scale_out=True)
|
||||
|
||||
assert paths >= SCALE_OUT_PATHS
|
||||
|
||||
|
||||
def test_render_routes_remain_enabled_for_render_server(monkeypatch):
|
||||
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
|
||||
|
||||
def test_render_routes_remain_enabled_for_render_server():
|
||||
paths = registered_paths(("render",))
|
||||
|
||||
assert paths >= RENDER_PATHS
|
||||
assert GENERATE_PATH not in paths
|
||||
|
||||
|
||||
def test_render_server_rejects_explicitly_disabled_scale_out_routes(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "0")
|
||||
|
||||
with pytest.raises(ValueError, match="VLLM_ENABLE_SCALE_OUT_ENDPOINTS=0"):
|
||||
registered_paths(("render",))
|
||||
|
||||
|
||||
def test_tokens_only_mode_enables_generate_routes_when_flag_is_unset(monkeypatch):
|
||||
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
|
||||
|
||||
def test_tokens_only_mode_enables_generate_routes_when_flag_is_unset():
|
||||
paths = registered_paths(("generate",), tokens_only=True)
|
||||
|
||||
assert paths >= {GENERATE_PATH, "/abort_requests"}
|
||||
|
||||
|
||||
def test_tokens_only_mode_rejects_explicitly_disabled_scale_out_routes(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "0")
|
||||
|
||||
with pytest.raises(ValueError, match="--tokens-only"):
|
||||
registered_paths(("generate",), tokens_only=True)
|
||||
|
||||
@@ -34,12 +34,9 @@ def server():
|
||||
"--enable-return-routed-experts",
|
||||
"--hf-overrides",
|
||||
'{"sliding_window": null}',
|
||||
"--enable-scale-out",
|
||||
]
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME,
|
||||
args,
|
||||
env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"},
|
||||
) as remote_server:
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
|
||||
@@ -37,13 +37,10 @@ def server():
|
||||
"4096",
|
||||
"--enforce-eager",
|
||||
"--no-enable-prefix-caching",
|
||||
"--enable-scale-out",
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME,
|
||||
args,
|
||||
env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"},
|
||||
) as remote_server:
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
|
||||
@@ -69,6 +69,7 @@ def server(request):
|
||||
# Option B: deterministic prefix caching
|
||||
# "--enable-prefix-caching",
|
||||
# "--deterministic-prefix-caching",
|
||||
"--enable-scale-out",
|
||||
]
|
||||
|
||||
extra_args = getattr(request, "param", None)
|
||||
@@ -79,11 +80,7 @@ def server(request):
|
||||
else [str(extra_args)]
|
||||
)
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME,
|
||||
args,
|
||||
env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"},
|
||||
) as remote_server:
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user