[Frontend] Gate scale-out endpoints behind opt-in flag (#54579)
Signed-off-by: Francisco Javier Arceo <farceo@redhat.com>
This commit is contained in:
@@ -58,6 +58,7 @@ def _enable_cohere_api(monkeypatch):
|
||||
flag inside the test body.
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_ENABLE_COHERE_API", "1")
|
||||
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "1")
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
@@ -402,11 +403,12 @@ class TestRenderEndpoint:
|
||||
assert "/cohere/v2/chat/render" in paths
|
||||
|
||||
def test_route_not_registered_when_flag_unset(self, monkeypatch):
|
||||
monkeypatch.delenv("VLLM_ENABLE_COHERE_API", raising=False)
|
||||
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
|
||||
app = FastAPI()
|
||||
attach_router(app)
|
||||
paths = [getattr(r, "path", None) for r in app.routes]
|
||||
assert "/cohere/v2/chat/render" not in paths
|
||||
assert "/cohere/v2/chat" in paths
|
||||
|
||||
def test_returns_generate_request_json(self):
|
||||
render_handler = _RenderHandler(_generate_request())
|
||||
|
||||
@@ -71,6 +71,7 @@ def server():
|
||||
env_dict = {
|
||||
"VLLM_CPU_KVCACHE_SPACE": "1",
|
||||
"VLLM_ENABLE_COHERE_API": "1",
|
||||
"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1",
|
||||
}
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server:
|
||||
|
||||
@@ -54,7 +54,11 @@ FORCE_WEATHER_TOOL = {"type": "function", "function": {"name": "get_weather"}}
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
with RemoteOpenAIServer(MODEL, ARGS) as remote_server:
|
||||
with RemoteOpenAIServer(
|
||||
MODEL,
|
||||
ARGS,
|
||||
env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"},
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
|
||||
@@ -29,7 +29,7 @@ def vision_server():
|
||||
"0",
|
||||
]
|
||||
|
||||
env_overrides: dict[str, str] = {}
|
||||
env_overrides = {"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"}
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
VISION_MODEL_NAME,
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from argparse import Namespace
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
|
||||
from vllm.entrypoints.launchers.api_server.routers import register_api_routers
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
RENDER_PATHS = {
|
||||
"/v1/chat/completions/render",
|
||||
"/v1/chat/completions/derender",
|
||||
"/v1/completions/render",
|
||||
"/v1/completions/derender",
|
||||
"/v1/messages/render",
|
||||
}
|
||||
GENERATE_PATH = "/inference/v1/generate"
|
||||
SCALE_OUT_PATHS = RENDER_PATHS | {GENERATE_PATH}
|
||||
|
||||
|
||||
def registered_paths(
|
||||
supported_tasks: tuple[SupportedTask, ...], *, tokens_only: bool = False
|
||||
) -> set[str]:
|
||||
app = FastAPI()
|
||||
args = Namespace(tokens_only=tokens_only, enable_fault_tolerance=False)
|
||||
app.state.args = args
|
||||
register_api_routers(args, app, supported_tasks)
|
||||
return {route.path for route in app.routes}
|
||||
|
||||
|
||||
def test_scale_out_routes_are_disabled_by_default(monkeypatch):
|
||||
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
|
||||
|
||||
paths = registered_paths(("generate",))
|
||||
|
||||
assert SCALE_OUT_PATHS.isdisjoint(paths)
|
||||
|
||||
|
||||
def test_disabled_scale_out_routes_are_logged(monkeypatch, caplog):
|
||||
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
|
||||
|
||||
with caplog.at_level("INFO", logger="vllm.entrypoints.scale_out.factories"):
|
||||
registered_paths(("generate",))
|
||||
|
||||
assert any(
|
||||
"VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1" in record.message
|
||||
for record in caplog.records
|
||||
)
|
||||
|
||||
|
||||
def test_scale_out_routes_can_be_enabled_for_generate_server(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "1")
|
||||
|
||||
paths = registered_paths(("generate",))
|
||||
|
||||
assert paths >= SCALE_OUT_PATHS
|
||||
|
||||
|
||||
def test_render_routes_remain_enabled_for_render_server(monkeypatch):
|
||||
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
|
||||
|
||||
paths = registered_paths(("render",))
|
||||
|
||||
assert paths >= RENDER_PATHS
|
||||
assert GENERATE_PATH not in paths
|
||||
|
||||
|
||||
def test_render_server_rejects_explicitly_disabled_scale_out_routes(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "0")
|
||||
|
||||
with pytest.raises(ValueError, match="VLLM_ENABLE_SCALE_OUT_ENDPOINTS=0"):
|
||||
registered_paths(("render",))
|
||||
|
||||
|
||||
def test_tokens_only_mode_enables_generate_routes_when_flag_is_unset(monkeypatch):
|
||||
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
|
||||
|
||||
paths = registered_paths(("generate",), tokens_only=True)
|
||||
|
||||
assert paths >= {GENERATE_PATH, "/abort_requests"}
|
||||
|
||||
|
||||
def test_tokens_only_mode_rejects_explicitly_disabled_scale_out_routes(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "0")
|
||||
|
||||
with pytest.raises(ValueError, match="--tokens-only"):
|
||||
registered_paths(("generate",), tokens_only=True)
|
||||
@@ -35,7 +35,11 @@ def server():
|
||||
"--hf-overrides",
|
||||
'{"sliding_window": null}',
|
||||
]
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME,
|
||||
args,
|
||||
env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"},
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
|
||||
@@ -39,7 +39,11 @@ def server():
|
||||
"--no-enable-prefix-caching",
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME,
|
||||
args,
|
||||
env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"},
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
|
||||
@@ -79,7 +79,11 @@ def server(request):
|
||||
else [str(extra_args)]
|
||||
)
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME,
|
||||
args,
|
||||
env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"},
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user