[Frontend] Replace VLLM_ENABLE_SCALE_OUT_ENDPOINTS with --enable-scale-out (#55176)

Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
Martin Hickey
2026-09-14 10:02:14 +00:00
committed by GitHub
parent 2c7ee87223
commit 934b1fcbcb
24 changed files with 130 additions and 215 deletions
+13 -10
View File
@@ -58,7 +58,6 @@ def _enable_cohere_api(monkeypatch):
flag inside the test body.
"""
monkeypatch.setenv("VLLM_ENABLE_COHERE_API", "1")
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "1")
# ----------------------------------------------------------------------
@@ -128,15 +127,19 @@ def _generate_request() -> GenerateRequest:
)
def _build_app(handler: _Handler | None) -> FastAPI:
def _build_app(handler: _Handler | None, *, enable_scale_out: bool = True) -> FastAPI:
app = FastAPI()
app.state.args = Namespace(enable_scale_out=enable_scale_out, tokens_only=False)
attach_router(app)
app.state.cohere_serving_chat_v2 = handler
return app
def _build_render_app(chat_handler, render_handler) -> FastAPI:
def _build_render_app(
chat_handler, render_handler, *, enable_scale_out: bool = True
) -> FastAPI:
app = FastAPI()
app.state.args = Namespace(enable_scale_out=enable_scale_out, tokens_only=False)
attach_router(app)
app.state.cohere_serving_chat_v2 = chat_handler
app.state.serving_render = render_handler
@@ -151,11 +154,13 @@ def _build_app_with_vllm_handlers(handler: _Handler | None) -> FastAPI:
into the ``CohereError`` wire shape.
"""
app = FastAPI()
attach_router(app)
app.state.cohere_serving_chat_v2 = handler
# ``validation_exception_handler`` reads ``req.app.state.args``; the
# real cli builds this via argparse.
app.state.args = Namespace(log_error_stack=False)
app.state.args = Namespace(
log_error_stack=False, enable_scale_out=True, tokens_only=False
)
attach_router(app)
app.state.cohere_serving_chat_v2 = handler
app.exception_handler(RequestValidationError)(validation_exception_handler)
app.exception_handler(HTTPException)(http_exception_handler)
app.exception_handler(VLLMError)(vllm_error_handler)
@@ -402,10 +407,8 @@ class TestRenderEndpoint:
paths = [getattr(r, "path", None) for r in app.routes]
assert "/cohere/v2/chat/render" in paths
def test_route_not_registered_when_flag_unset(self, monkeypatch):
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
app = FastAPI()
attach_router(app)
def test_route_not_registered_when_flag_unset(self):
app = _build_app(handler=None, enable_scale_out=False)
paths = [getattr(r, "path", None) for r in app.routes]
assert "/cohere/v2/chat/render" not in paths
assert "/cohere/v2/chat" in paths
+2 -1
View File
@@ -57,6 +57,8 @@ def server():
# the conversation to surface a thinking block (SmolLM2 doesn't
# emit Cohere-style reasoning tokens out of the box).
"--no-cohere-is-reasoning-model",
# Registers /cohere/v2/chat/render alongside /cohere/v2/chat.
"--enable-scale-out",
]
# Cap the CPU KV-cache pool the vLLM CPU backend reserves at
@@ -71,7 +73,6 @@ def server():
env_dict = {
"VLLM_CPU_KVCACHE_SPACE": "1",
"VLLM_ENABLE_COHERE_API": "1",
"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1",
}
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server: