diff --git a/docs/serving/online_serving/README.md b/docs/serving/online_serving/README.md index 5c93ca1787..1f5d4bdb6a 100644 --- a/docs/serving/online_serving/README.md +++ b/docs/serving/online_serving/README.md @@ -121,12 +121,7 @@ For further details on profiling vLLM, please refer to [this page](../../contrib ## Scale-Out APIs -Scale-out APIs are disabled by default on `vllm serve`. The environment -variable accepts only `0` or `1`; set `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1` to -register the endpoints below. The -dedicated `vllm launch render` and `vllm serve --tokens-only` modes are explicit -opt-ins and enable their required endpoints when the variable is unset; an -explicit value of `0` is rejected for those modes. +Scale-out APIs are disabled by default on `vllm serve`. Set `--enable-scale-out` to register the endpoints below. The dedicated `vllm launch render` and `vllm serve --tokens-only` modes always register their required endpoints regardless of `--enable-scale-out`. ### Tokens IN <> Tokens OUT APIs diff --git a/docs/serving/online_serving/derenderer.md b/docs/serving/online_serving/derenderer.md index d536498f5c..298510062a 100644 --- a/docs/serving/online_serving/derenderer.md +++ b/docs/serving/online_serving/derenderer.md @@ -60,8 +60,7 @@ Launch the two servers first: ```bash vllm launch render meta-llama/Llama-3.2-1B-Instruct --port 8100 -VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 vllm serve \ - meta-llama/Llama-3.2-1B-Instruct --tokens-only --port 8200 +vllm serve meta-llama/Llama-3.2-1B-Instruct --tokens-only --port 8200 ``` ```python diff --git a/docs/serving/online_serving/renderer.md b/docs/serving/online_serving/renderer.md index 1f5e893570..9be23142c4 100644 --- a/docs/serving/online_serving/renderer.md +++ b/docs/serving/online_serving/renderer.md @@ -7,16 +7,14 @@ Our renderer API is designed to disaggregate the render phase(preprocessing) and - Tokens-in / tokens-out engine: Make the engine a pure token-in / token-out service, decoupled from request preprocessing. The dedicated `vllm launch render` server always exposes the `/render` and -`/derender` endpoints when `VLLM_ENABLE_SCALE_OUT_ENDPOINTS` is unset or set to -`1`. An explicit value of `0` conflicts with the renderer command and is -rejected at startup. +`/derender` endpoints. Scale-out endpoints, including `/render`, `/derender`, and `/inference/v1/generate`, are disabled by default on a standard inference server. To expose them with `vllm serve`, opt in explicitly: ```bash -VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 vllm serve +vllm serve --enable-scale-out ``` ## API Reference diff --git a/docs/usage/security.md b/docs/usage/security.md index 559f587026..3c7f1e3f9b 100644 --- a/docs/usage/security.md +++ b/docs/usage/security.md @@ -158,16 +158,16 @@ When `--api-key` is configured, the following endpoints require Bearer token aut - `/v1/models` - List available models - `/v1/chat/completions` - Chat completions - `/v1/chat/completions/batch` - Batch chat completions -- `/v1/chat/completions/render` - Render chat completion requests (available on `vllm serve` only when `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1`, or on `vllm launch render` unless explicitly disabled) -- `/v1/chat/completions/derender` - Derender chat completion requests (available on `vllm serve` only when `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1`, or on `vllm launch render` unless explicitly disabled) +- `/v1/chat/completions/render` - Render chat completion requests (available on `vllm serve` only when `--enable-scale-out` is set, or on `vllm launch render`) +- `/v1/chat/completions/derender` - Derender chat completion requests (available on `vllm serve` only when `--enable-scale-out` is set, or on `vllm launch render`) - `/v1/completions` - Text completions -- `/v1/completions/render` - Render completion requests (available on `vllm serve` only when `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1`, or on `vllm launch render` unless explicitly disabled) -- `/v1/completions/derender` - Derender completion requests (available on `vllm serve` only when `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1`, or on `vllm launch render` unless explicitly disabled) +- `/v1/completions/render` - Render completion requests (available on `vllm serve` only when `--enable-scale-out` is set, or on `vllm launch render`) +- `/v1/completions/derender` - Derender completion requests (available on `vllm serve` only when `--enable-scale-out` is set, or on `vllm launch render`) - `/v1/embeddings` - Generate embeddings - `/v1/audio/transcriptions` - Audio transcription - `/v1/audio/translations` - Audio translation - `/v1/messages` - Anthropic-compatible messages API -- `/v1/messages/render` - Render Anthropic-compatible messages (available on `vllm serve` only when `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1`, or on `vllm launch render` unless explicitly disabled) +- `/v1/messages/render` - Render Anthropic-compatible messages (available on `vllm serve` only when `--enable-scale-out` is set, or on `vllm launch render`) - `/v1/messages/count_tokens` - Count tokens for Anthropic messages - `/v1/responses` - Create a response - `/v1/responses/render` - Render a self-contained response request (available on `vllm serve` only when `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1`, or on `vllm launch render` unless explicitly disabled) @@ -177,8 +177,8 @@ When `--api-key` is configured, the following endpoints require Bearer token aut - `/v1/rerank` - Reranking API - `/v1/load_lora_adapter` - Load a LoRA adapter (can alter model behavior; only available when `--enable-lora` is set and `VLLM_ALLOW_RUNTIME_LORA_UPDATING=True`) - `/v1/unload_lora_adapter` - Unload a LoRA adapter (can alter model behavior; only available when `--enable-lora` is set and `VLLM_ALLOW_RUNTIME_LORA_UPDATING=True`) -- `/inference/v1/generate` - Generate completions (available when `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1`, or with `--tokens-only` unless explicitly disabled) -- `/cohere/v2/chat/render` - Render Cohere Chat v2 requests (requires both `VLLM_ENABLE_COHERE_API=1` and `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1`) +- `/inference/v1/generate` - Generate completions (available when `--enable-scale-out` is set, or with `--tokens-only`) +- `/cohere/v2/chat/render` - Render Cohere Chat v2 requests (requires `VLLM_ENABLE_COHERE_API=1` and either `--enable-scale-out` or `--tokens-only`) - `/v2/embed` - Cohere Embed API - `/v2/rerank` - Cohere Rerank API @@ -206,11 +206,11 @@ The following endpoints **do not require authentication** even when `--api-key` - `/init_weight_transfer_engine` - Initialize weight transfer engine for RLHF - `/update_weights` - Update model weights (can alter model behavior) - `/get_world_size` - Get distributed world size -- `/abort_requests` - Abort in-flight requests (available with `--tokens-only` unless `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=0`) +- `/abort_requests` - Abort in-flight requests (available with `--tokens-only`) **Utility endpoints:** -- `/tokenize` - Tokenize text (not disabled by `VLLM_ENABLE_SCALE_OUT_ENDPOINTS`) +- `/tokenize` - Tokenize text (not gated by `--enable-scale-out`) - `/detokenize` - Detokenize tokens - `/health` - Health check - `/ping` - SageMaker health check diff --git a/examples/scale_out/example_mm_serve.py b/examples/scale_out/example_mm_serve.py index 157a32d3be..558a4f9302 100644 --- a/examples/scale_out/example_mm_serve.py +++ b/examples/scale_out/example_mm_serve.py @@ -13,9 +13,9 @@ zero client-side transformation. Launch the server first: - VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 \ - vllm serve Qwen/Qwen3-VL-2B-Instruct \ - --dtype bfloat16 --max-model-len 4096 --enforce-eager + vllm serve Qwen/Qwen3-VL-2B-Instruct \ + --dtype bfloat16 --max-model-len 4096 --enforce-eager \ + --enable-scale-out Then run this script: diff --git a/examples/scale_out/token_generation_client.py b/examples/scale_out/token_generation_client.py index 922018f43d..a4792a68f9 100644 --- a/examples/scale_out/token_generation_client.py +++ b/examples/scale_out/token_generation_client.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Call a token-in/token-out server started with: -VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 vllm serve Qwen/Qwen3-0.6B +vllm serve Qwen/Qwen3-0.6B --enable-scale-out """ import httpx diff --git a/rust/src/cmd/src/cli.rs b/rust/src/cmd/src/cli.rs index f7fcf30022..70b28018a3 100644 --- a/rust/src/cmd/src/cli.rs +++ b/rust/src/cmd/src/cli.rs @@ -329,6 +329,15 @@ pub struct SharedRuntimeArgs { #[serde(default)] pub enable_request_id_headers: bool, + /// Register the scale-out `/inference/v1/generate` endpoint. + #[arg( + long, + default_missing_value = "true", + num_args = 0..=1 + )] + #[serde(default)] + pub enable_scale_out: bool, + /// If provided, the server will require one of these keys to be presented /// in the Authorization header. #[educe(Debug(ignore))] @@ -577,6 +586,7 @@ impl SharedRuntimeArgs { enable_log_requests: self.enable_log_requests, enable_prompt_tokens_details: self.enable_prompt_tokens_details, enable_request_id_headers: self.enable_request_id_headers, + enable_scale_out: self.enable_scale_out, } } diff --git a/rust/src/cmd/src/cli/tests.rs b/rust/src/cmd/src/cli/tests.rs index 884da135d7..411d256295 100644 --- a/rust/src/cmd/src/cli/tests.rs +++ b/rust/src/cmd/src/cli/tests.rs @@ -195,6 +195,7 @@ fn serve_args_forward_python_flags_with_separator() { enable_log_requests: false, enable_prompt_tokens_details: false, enable_request_id_headers: false, + enable_scale_out: false, disable_log_stats: false, served_model_name: [], allowed_origins: JsonStringList( @@ -1030,6 +1031,7 @@ fn frontend_args_accept_json() { enable_log_requests: false, enable_prompt_tokens_details: false, enable_request_id_headers: false, + enable_scale_out: false, disable_log_stats: false, served_model_name: [], allowed_origins: JsonStringList( @@ -1629,6 +1631,7 @@ fn serve_args_accept_handshake_aliases() { enable_log_requests: false, enable_prompt_tokens_details: false, enable_request_id_headers: false, + enable_scale_out: false, disable_log_stats: false, served_model_name: [], allowed_origins: JsonStringList( @@ -1780,6 +1783,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() { enable_log_requests: false, enable_prompt_tokens_details: false, enable_request_id_headers: false, + enable_scale_out: false, }, cors: CorsConfig { allow_origins: [ @@ -1868,6 +1872,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() { enable_log_requests: false, enable_prompt_tokens_details: false, enable_request_id_headers: false, + enable_scale_out: false, }, cors: CorsConfig { allow_origins: [ @@ -1977,6 +1982,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present enable_log_requests: false, enable_prompt_tokens_details: false, enable_request_id_headers: false, + enable_scale_out: false, }, cors: CorsConfig { allow_origins: [ diff --git a/rust/src/server/src/config.rs b/rust/src/server/src/config.rs index 4928cddf31..0f0deac86d 100644 --- a/rust/src/server/src/config.rs +++ b/rust/src/server/src/config.rs @@ -55,6 +55,8 @@ pub struct ApiServerOptions { pub enable_prompt_tokens_details: bool, /// When `true`, set `X-Request-Id` on every HTTP response. pub enable_request_id_headers: bool, + /// When `true`, register the scale-out `/inference/v1/generate` route. + pub enable_scale_out: bool, } /// CORS settings mirroring Python's `CORSMiddleware`; the default is permissive. diff --git a/rust/src/server/src/routes.rs b/rust/src/server/src/routes.rs index e7e91938bb..41e4987882 100644 --- a/rust/src/server/src/routes.rs +++ b/rust/src/server/src/routes.rs @@ -46,33 +46,14 @@ fn runtime_lora_updating_enabled() -> bool { .is_some_and(|value| matches!(value.trim().to_lowercase().as_str(), "1" | "true")) } -fn parse_scale_out_endpoints_flag(value: Option<&str>) -> Result { - let Some(value) = value else { - return Ok(false); - }; - if value.is_empty() { - return Ok(false); - } - - match value.trim() { - "0" => Ok(false), - "1" => Ok(true), - _ => Err("VLLM_ENABLE_SCALE_OUT_ENDPOINTS must be 0 or 1".to_string()), - } -} - -fn scale_out_endpoints_enabled() -> bool { - let value = std::env::var("VLLM_ENABLE_SCALE_OUT_ENDPOINTS").ok(); - parse_scale_out_endpoints_flag(value.as_deref()).unwrap_or_else(|message| panic!("{message}")) -} - /// Build the minimal OpenAI-compatible router for one configured model. pub fn build_router(state: Arc) -> Router { + let scale_out_endpoints_enabled = state.api_server_options.enable_scale_out; build_router_with_options( state, server_dev_mode_enabled(), runtime_lora_updating_enabled(), - scale_out_endpoints_enabled(), + scale_out_endpoints_enabled, ) } @@ -121,10 +102,7 @@ fn build_router_with_options( if scale_out_endpoints_enabled { router = router.route("/inference/v1/generate", post(inference::generate)); } else { - info!( - "scale-out endpoints are disabled; set \ - VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 to enable them" - ); + info!("scale-out endpoints are disabled; pass --enable-scale-out to enable them"); } if runtime_lora_updating_enabled { diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 2851ec1e88..5ee2b51f3f 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -54,8 +54,7 @@ use zeromq::{DealerSocket, PushSocket, ZmqMessage}; use super::{ build_router, build_router_with_dev_mode, build_router_with_dev_mode_and_lora, - build_router_with_scale_out_endpoints, parse_scale_out_endpoints_flag, - render::build_router as build_render_router, + build_router_with_scale_out_endpoints, render::build_router as build_render_router, }; use crate::config::{ApiServerOptions, CorsConfig, LoraModulePath}; use crate::lora::LoadLoraError; @@ -731,23 +730,36 @@ async fn test_app_with_dev_mode(dev_mode_enabled: bool) -> axum::Router { ) } -#[test] -fn scale_out_flag_accepts_unset_empty_zero_one_and_whitespace() { - assert_eq!(parse_scale_out_endpoints_flag(None), Ok(false)); - assert_eq!(parse_scale_out_endpoints_flag(Some("")), Ok(false)); - assert_eq!(parse_scale_out_endpoints_flag(Some("0")), Ok(false)); - assert_eq!(parse_scale_out_endpoints_flag(Some("1")), Ok(true)); - assert_eq!(parse_scale_out_endpoints_flag(Some(" 1 ")), Ok(true)); -} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn scale_out_generate_route_follows_api_server_options() { + let (chat, _engine_task) = test_models_with_engine_outputs_and_backend( + b"engine-scale-out-api-server-options", + default_stream_output_specs(), + Arc::new(FakeChatBackend::new()), + ) + .await; + let state = Arc::new( + AppState::new(vec!["model".to_string()], chat).with_api_server_options(ApiServerOptions { + enable_scale_out: true, + ..Default::default() + }), + ); + let mut app = build_router(state); -#[test] -fn scale_out_flag_rejects_values_other_than_zero_or_one() { - for value in ["-1", "2", "01", "+1", "invalid", " "] { - assert_eq!( - parse_scale_out_endpoints_flag(Some(value)), - Err("VLLM_ENABLE_SCALE_OUT_ENDPOINTS must be 0 or 1".to_string()) - ); - } + let response = app + .call( + Request::builder() + .method("POST") + .uri("/inference/v1/generate") + .header("content-type", "application/json") + .body(Body::from("{}")) + .expect("build request"), + ) + .await + .expect("call app"); + + assert_ne!(response.status(), StatusCode::NOT_FOUND); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/tests/entrypoints/cohere/test_api_router.py b/tests/entrypoints/cohere/test_api_router.py index a90e93b6b1..94324dbed7 100644 --- a/tests/entrypoints/cohere/test_api_router.py +++ b/tests/entrypoints/cohere/test_api_router.py @@ -58,7 +58,6 @@ def _enable_cohere_api(monkeypatch): flag inside the test body. """ monkeypatch.setenv("VLLM_ENABLE_COHERE_API", "1") - monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "1") # ---------------------------------------------------------------------- @@ -128,15 +127,19 @@ def _generate_request() -> GenerateRequest: ) -def _build_app(handler: _Handler | None) -> FastAPI: +def _build_app(handler: _Handler | None, *, enable_scale_out: bool = True) -> FastAPI: app = FastAPI() + app.state.args = Namespace(enable_scale_out=enable_scale_out, tokens_only=False) attach_router(app) app.state.cohere_serving_chat_v2 = handler return app -def _build_render_app(chat_handler, render_handler) -> FastAPI: +def _build_render_app( + chat_handler, render_handler, *, enable_scale_out: bool = True +) -> FastAPI: app = FastAPI() + app.state.args = Namespace(enable_scale_out=enable_scale_out, tokens_only=False) attach_router(app) app.state.cohere_serving_chat_v2 = chat_handler app.state.serving_render = render_handler @@ -151,11 +154,13 @@ def _build_app_with_vllm_handlers(handler: _Handler | None) -> FastAPI: into the ``CohereError`` wire shape. """ app = FastAPI() - attach_router(app) - app.state.cohere_serving_chat_v2 = handler # ``validation_exception_handler`` reads ``req.app.state.args``; the # real cli builds this via argparse. - app.state.args = Namespace(log_error_stack=False) + app.state.args = Namespace( + log_error_stack=False, enable_scale_out=True, tokens_only=False + ) + attach_router(app) + app.state.cohere_serving_chat_v2 = handler app.exception_handler(RequestValidationError)(validation_exception_handler) app.exception_handler(HTTPException)(http_exception_handler) app.exception_handler(VLLMError)(vllm_error_handler) @@ -402,10 +407,8 @@ class TestRenderEndpoint: paths = [getattr(r, "path", None) for r in app.routes] assert "/cohere/v2/chat/render" in paths - def test_route_not_registered_when_flag_unset(self, monkeypatch): - monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False) - app = FastAPI() - attach_router(app) + def test_route_not_registered_when_flag_unset(self): + app = _build_app(handler=None, enable_scale_out=False) paths = [getattr(r, "path", None) for r in app.routes] assert "/cohere/v2/chat/render" not in paths assert "/cohere/v2/chat" in paths diff --git a/tests/entrypoints/cohere/test_chat_v2.py b/tests/entrypoints/cohere/test_chat_v2.py index 36acecb622..4f07d3d873 100644 --- a/tests/entrypoints/cohere/test_chat_v2.py +++ b/tests/entrypoints/cohere/test_chat_v2.py @@ -57,6 +57,8 @@ def server(): # the conversation to surface a thinking block (SmolLM2 doesn't # emit Cohere-style reasoning tokens out of the box). "--no-cohere-is-reasoning-model", + # Registers /cohere/v2/chat/render alongside /cohere/v2/chat. + "--enable-scale-out", ] # Cap the CPU KV-cache pool the vLLM CPU backend reserves at @@ -71,7 +73,6 @@ def server(): env_dict = { "VLLM_CPU_KVCACHE_SPACE": "1", "VLLM_ENABLE_COHERE_API": "1", - "VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1", } with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server: diff --git a/tests/entrypoints/scale_out/derender/test_derender_parity.py b/tests/entrypoints/scale_out/derender/test_derender_parity.py index 5b313464da..adece09c0a 100644 --- a/tests/entrypoints/scale_out/derender/test_derender_parity.py +++ b/tests/entrypoints/scale_out/derender/test_derender_parity.py @@ -34,6 +34,7 @@ ARGS = [ "hermes", "--reasoning-parser", "deepseek_r1", + "--enable-scale-out", ] TOOLS = [ @@ -54,11 +55,7 @@ FORCE_WEATHER_TOOL = {"type": "function", "function": {"name": "get_weather"}} @pytest.fixture(scope="module") def server(): - with RemoteOpenAIServer( - MODEL, - ARGS, - env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"}, - ) as remote_server: + with RemoteOpenAIServer(MODEL, ARGS) as remote_server: yield remote_server diff --git a/tests/entrypoints/scale_out/render/test_render_multimodal.py b/tests/entrypoints/scale_out/render/test_render_multimodal.py index 70bbc95016..c6a6931948 100644 --- a/tests/entrypoints/scale_out/render/test_render_multimodal.py +++ b/tests/entrypoints/scale_out/render/test_render_multimodal.py @@ -27,15 +27,10 @@ def vision_server(): "1", "--limit-mm-per-prompt.video", "0", + "--enable-scale-out", ] - env_overrides = {"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"} - - with RemoteOpenAIServer( - VISION_MODEL_NAME, - args, - env_dict=env_overrides, - ) as remote_server: + with RemoteOpenAIServer(VISION_MODEL_NAME, args) as remote_server: yield remote_server diff --git a/tests/entrypoints/scale_out/test_factories.py b/tests/entrypoints/scale_out/test_factories.py index 9811bca8a2..6d8335395e 100644 --- a/tests/entrypoints/scale_out/test_factories.py +++ b/tests/entrypoints/scale_out/test_factories.py @@ -3,7 +3,6 @@ from argparse import Namespace -import pytest from fastapi import FastAPI from vllm.entrypoints.launchers.api_server.routers import register_api_routers @@ -22,69 +21,49 @@ SCALE_OUT_PATHS = RENDER_PATHS | {GENERATE_PATH} def registered_paths( - supported_tasks: tuple[SupportedTask, ...], *, tokens_only: bool = False + supported_tasks: tuple[SupportedTask, ...], + *, + tokens_only: bool = False, + enable_scale_out: bool = False, ) -> set[str]: app = FastAPI() - args = Namespace(tokens_only=tokens_only, enable_fault_tolerance=False) + args = Namespace( + tokens_only=tokens_only, + enable_scale_out=enable_scale_out, + enable_fault_tolerance=False, + ) app.state.args = args register_api_routers(args, app, supported_tasks) return {route.path for route in app.routes} -def test_scale_out_routes_are_disabled_by_default(monkeypatch): - monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False) - +def test_scale_out_routes_are_disabled_by_default(): paths = registered_paths(("generate",)) assert SCALE_OUT_PATHS.isdisjoint(paths) -def test_disabled_scale_out_routes_are_logged(monkeypatch, caplog): - monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False) - +def test_disabled_scale_out_routes_are_logged(caplog): with caplog.at_level("INFO", logger="vllm.entrypoints.scale_out.factories"): registered_paths(("generate",)) - assert any( - "VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1" in record.message - for record in caplog.records - ) + assert any("--enable-scale-out" in record.message for record in caplog.records) -def test_scale_out_routes_can_be_enabled_for_generate_server(monkeypatch): - monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "1") - - paths = registered_paths(("generate",)) +def test_scale_out_routes_can_be_enabled_for_generate_server(): + paths = registered_paths(("generate",), enable_scale_out=True) assert paths >= SCALE_OUT_PATHS -def test_render_routes_remain_enabled_for_render_server(monkeypatch): - monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False) - +def test_render_routes_remain_enabled_for_render_server(): paths = registered_paths(("render",)) assert paths >= RENDER_PATHS assert GENERATE_PATH not in paths -def test_render_server_rejects_explicitly_disabled_scale_out_routes(monkeypatch): - monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "0") - - with pytest.raises(ValueError, match="VLLM_ENABLE_SCALE_OUT_ENDPOINTS=0"): - registered_paths(("render",)) - - -def test_tokens_only_mode_enables_generate_routes_when_flag_is_unset(monkeypatch): - monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False) - +def test_tokens_only_mode_enables_generate_routes_when_flag_is_unset(): paths = registered_paths(("generate",), tokens_only=True) assert paths >= {GENERATE_PATH, "/abort_requests"} - - -def test_tokens_only_mode_rejects_explicitly_disabled_scale_out_routes(monkeypatch): - monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "0") - - with pytest.raises(ValueError, match="--tokens-only"): - registered_paths(("generate",), tokens_only=True) diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_return_routed_experts.py b/tests/entrypoints/scale_out/token_in_token_out/test_return_routed_experts.py index 963e416e7e..e7527a071b 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_return_routed_experts.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_return_routed_experts.py @@ -34,12 +34,9 @@ def server(): "--enable-return-routed-experts", "--hf-overrides", '{"sliding_window": null}', + "--enable-scale-out", ] - with RemoteOpenAIServer( - MODEL_NAME, - args, - env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"}, - ) as remote_server: + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server: yield remote_server diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_serving_multimodal_tokens.py b/tests/entrypoints/scale_out/token_in_token_out/test_serving_multimodal_tokens.py index 9491623fd1..892d3c8285 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_serving_multimodal_tokens.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_serving_multimodal_tokens.py @@ -37,13 +37,10 @@ def server(): "4096", "--enforce-eager", "--no-enable-prefix-caching", + "--enable-scale-out", ] - with RemoteOpenAIServer( - MODEL_NAME, - args, - env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"}, - ) as remote_server: + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server: yield remote_server diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py b/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py index fb741df061..1c349e8d67 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py @@ -69,6 +69,7 @@ def server(request): # Option B: deterministic prefix caching # "--enable-prefix-caching", # "--deterministic-prefix-caching", + "--enable-scale-out", ] extra_args = getattr(request, "param", None) @@ -79,11 +80,7 @@ def server(request): else [str(extra_args)] ) - with RemoteOpenAIServer( - MODEL_NAME, - args, - env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"}, - ) as remote_server: + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server: yield remote_server diff --git a/tests/test_envs.py b/tests/test_envs.py index aaa53d2a54..1ef40cde0c 100644 --- a/tests/test_envs.py +++ b/tests/test_envs.py @@ -43,41 +43,6 @@ def test_api_key_is_not_compile_factor(monkeypatch: pytest.MonkeyPatch): assert "VLLM_API_KEY" not in envs.compile_factors() -def test_scale_out_endpoints_flag_is_runtime_only(monkeypatch: pytest.MonkeyPatch): - monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "1") - - assert envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS is True - assert "VLLM_ENABLE_SCALE_OUT_ENDPOINTS" not in envs.compile_factors() - - -def test_scale_out_endpoints_flag_distinguishes_unset_from_disabled( - monkeypatch: pytest.MonkeyPatch, -): - monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False) - assert envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS is None - - monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "0") - assert envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS is False - - -def test_scale_out_endpoints_flag_treats_empty_as_unset( - monkeypatch: pytest.MonkeyPatch, -): - monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "") - - assert envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS is None - - -@pytest.mark.parametrize("value", ["-1", "2", "01", "+1", "invalid", " "]) -def test_scale_out_endpoints_flag_rejects_values_other_than_zero_or_one( - monkeypatch: pytest.MonkeyPatch, value: str -): - monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", value) - - with pytest.raises(ValueError, match="must be 0 or 1"): - _ = envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS - - def test_p2p_side_channel_defaults_and_override(monkeypatch: pytest.MonkeyPatch): monkeypatch.delenv("VLLM_P2P_SIDE_CHANNEL_HOST", raising=False) monkeypatch.delenv("VLLM_P2P_SIDE_CHANNEL_PORT", raising=False) diff --git a/vllm/entrypoints/cohere/api_router.py b/vllm/entrypoints/cohere/api_router.py index 445964e6aa..eaac36f133 100644 --- a/vllm/entrypoints/cohere/api_router.py +++ b/vllm/entrypoints/cohere/api_router.py @@ -17,9 +17,10 @@ log) and vLLM continues to boot normally. Even when the SDK is installed, :func:`attach_router` also requires ``VLLM_ENABLE_COHERE_API=1`` in the environment before it will expose -the chat route. The render route additionally requires -``VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1``. These gates keep unrelated -deployments from accidentally exposing the APIs. +the chat route. The render route additionally requires scale-out +endpoints to be enabled, i.e. ``--enable-scale-out`` or +``--tokens-only``. These gates keep unrelated deployments from +accidentally exposing the APIs. Note: the handlers must live at module scope (not inside ``attach_router``) so that FastAPI's ``typing.get_type_hints`` resolves @@ -306,6 +307,7 @@ def attach_router(app: FastAPI) -> None: ) return app.include_router(router) - if envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS: + args = getattr(app.state, "args", None) + if getattr(args, "enable_scale_out", False) or getattr(args, "tokens_only", False): app.include_router(render_router) app.add_middleware(CohereErrorEnvelopeMiddleware) diff --git a/vllm/entrypoints/launchers/cli_args.py b/vllm/entrypoints/launchers/cli_args.py index 0da96b43e1..89af22cdc8 100644 --- a/vllm/entrypoints/launchers/cli_args.py +++ b/vllm/entrypoints/launchers/cli_args.py @@ -181,6 +181,13 @@ class BaseFrontendArgs: If set to True, only enable the Tokens In<>Out endpoint. This is intended for use in a Disaggregated Everything setup. """ + enable_scale_out: bool = False + """ + If set to True, register the scale-out endpoints (`/render`, `/derender`, + and `/inference/v1/generate`) on `vllm serve`. Has no effect on + `vllm launch render` or `vllm serve --tokens-only`, which always register + their required endpoints regardless of this flag. + """ fingerprint_mode: Literal["full", "hash", "custom", "none"] = "full" """Controls the ``system_fingerprint`` field on responses. diff --git a/vllm/entrypoints/scale_out/factories.py b/vllm/entrypoints/scale_out/factories.py index 30c329c51f..bd48c709dd 100644 --- a/vllm/entrypoints/scale_out/factories.py +++ b/vllm/entrypoints/scale_out/factories.py @@ -5,7 +5,6 @@ from typing import TYPE_CHECKING from fastapi import FastAPI -from vllm import envs from vllm.engine.protocol import EngineClient from vllm.logger import init_logger from vllm.tasks import SupportedTask @@ -67,22 +66,15 @@ def register_scale_out_api_routers( app: FastAPI, supported_tasks: tuple["SupportedTask", ...], ): - dedicated_mode = None - if "render" in supported_tasks: - dedicated_mode = "`vllm launch render`" - elif getattr(app.state.args, "tokens_only", False): - dedicated_mode = "`--tokens-only`" - - enabled = envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS - if dedicated_mode is not None and enabled is False: - raise ValueError( - "`VLLM_ENABLE_SCALE_OUT_ENDPOINTS=0` conflicts with " - f"{dedicated_mode}; unset it or set it to 1" - ) - if dedicated_mode is None and not enabled: + args = getattr(app.state, "args", None) + # `vllm launch render` and `vllm serve --tokens-only` are dedicated to + # serving these endpoints, so they always register them regardless of + # `--enable-scale-out`. + dedicated_mode = "render" in supported_tasks or getattr(args, "tokens_only", False) + enabled = dedicated_mode or getattr(args, "enable_scale_out", False) + if not enabled: logger.info( - "Scale-out endpoints are disabled. Set " - "VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 to enable them." + "Scale-out endpoints are disabled. Set --enable-scale-out to enable them." ) return diff --git a/vllm/envs.py b/vllm/envs.py index 0269a96409..6158f86f89 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -261,7 +261,6 @@ if TYPE_CHECKING: VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE: bool = True VLLM_ENABLE_RESPONSES_API_STORE: bool = False VLLM_ENABLE_COHERE_API: bool = False - VLLM_ENABLE_SCALE_OUT_ENDPOINTS: bool | None = None VLLM_HAS_FLASHINFER_CUBIN: bool = False VLLM_ROCM_FP8_MFMA_PAGE_ATTN: bool = False VLLM_ALLREDUCE_USE_SYMM_MEM: bool = True @@ -352,15 +351,6 @@ def maybe_convert_bool(value: str | None) -> bool | None: return bool(int(value)) -def maybe_convert_scale_out_endpoints(value: str | None) -> bool | None: - if value is None: - return None - normalized = value.strip() - if normalized not in ("0", "1"): - raise ValueError("VLLM_ENABLE_SCALE_OUT_ENDPOINTS must be 0 or 1") - return maybe_convert_bool(normalized) - - def maybe_convert_json_str_or_file(value: str | None) -> dict[str, Any] | None: if value is None: return None @@ -1858,12 +1848,6 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_ENABLE_COHERE_API": lambda: bool( int(os.getenv("VLLM_ENABLE_COHERE_API", "0")) ), - # If set to 1, expose the scale-out endpoints on `vllm serve`. - # The dedicated `vllm launch render` server exposes render and derender - # endpoints when this variable is unset or set to 1. - "VLLM_ENABLE_SCALE_OUT_ENDPOINTS": lambda: maybe_convert_scale_out_endpoints( - os.getenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS") or None - ), # If set, use the fp8 mfma in rocm paged attention. "VLLM_ROCM_FP8_MFMA_PAGE_ATTN": lambda: bool( int(os.getenv("VLLM_ROCM_FP8_MFMA_PAGE_ATTN", "0")) @@ -2289,7 +2273,6 @@ def compile_factors() -> dict[str, object]: "VLLM_CONFIG_ROOT", "LD_LIBRARY_PATH", "VLLM_SERVER_DEV_MODE", - "VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "VLLM_DP_MASTER_IP", "VLLM_DP_MASTER_PORT", "VLLM_NIXL_SIDE_CHANNEL_HOST",