From c00091e026709b149d80246a18928f514871c160 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Wed, 2 Sep 2026 02:31:14 -0400 Subject: [PATCH] [Frontend] Gate scale-out endpoints behind opt-in flag (#54579) Signed-off-by: Francisco Javier Arceo --- docs/serving/online_serving/README.md | 7 ++ docs/serving/online_serving/derenderer.md | 8 ++ docs/serving/online_serving/renderer.md | 13 ++ docs/usage/security.md | 16 +-- examples/scale_out/example_mm_serve.py | 3 +- examples/scale_out/token_generation_client.py | 5 + rust/src/server/src/routes.rs | 46 ++++++- rust/src/server/src/routes/tests.rs | 119 +++++++++++++++--- tests/entrypoints/cohere/test_api_router.py | 4 +- tests/entrypoints/cohere/test_chat_v2.py | 1 + .../derender/test_derender_parity.py | 6 +- .../render/test_render_multimodal.py | 2 +- tests/entrypoints/scale_out/test_factories.py | 89 +++++++++++++ .../test_return_routed_experts.py | 6 +- .../test_serving_multimodal_tokens.py | 6 +- .../token_in_token_out/test_serving_tokens.py | 6 +- tests/test_envs.py | 35 ++++++ vllm/entrypoints/cohere/api_router.py | 13 +- vllm/entrypoints/scale_out/factories.py | 23 ++++ vllm/envs.py | 17 +++ 20 files changed, 386 insertions(+), 39 deletions(-) create mode 100644 tests/entrypoints/scale_out/test_factories.py diff --git a/docs/serving/online_serving/README.md b/docs/serving/online_serving/README.md index 90ff7a3e3d..65c67e6a5b 100644 --- a/docs/serving/online_serving/README.md +++ b/docs/serving/online_serving/README.md @@ -121,6 +121,13 @@ For further details on profiling vLLM, please refer to [this page](../../contrib ## Scale-Out APIs +Scale-out APIs are disabled by default on `vllm serve`. The environment +variable accepts only `0` or `1`; set `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1` to +register the endpoints below. The +dedicated `vllm launch render` and `vllm serve --tokens-only` modes are explicit +opt-ins and enable their required endpoints when the variable is unset; an +explicit value of `0` is rejected for those modes. + ### Tokens IN <> Tokens OUT APIs - `/inference/v1/generate` - Generate completions diff --git a/docs/serving/online_serving/derenderer.md b/docs/serving/online_serving/derenderer.md index e55cd31094..d536498f5c 100644 --- a/docs/serving/online_serving/derenderer.md +++ b/docs/serving/online_serving/derenderer.md @@ -56,6 +56,14 @@ Oversized payloads are rejected with a `400` before any `tokenizer.decode()` or The example below drives the full `render → generate → derender` round trip for a chat request against a GPU less render server (`/render`, `/derender`) and a token-in / token-out engine (`/inference/v1/generate`). +Launch the two servers first: + +```bash +vllm launch render meta-llama/Llama-3.2-1B-Instruct --port 8100 +VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 vllm serve \ + meta-llama/Llama-3.2-1B-Instruct --tokens-only --port 8200 +``` + ```python import httpx diff --git a/docs/serving/online_serving/renderer.md b/docs/serving/online_serving/renderer.md index 517d50f2d3..5469490953 100644 --- a/docs/serving/online_serving/renderer.md +++ b/docs/serving/online_serving/renderer.md @@ -6,6 +6,19 @@ Our renderer API is designed to disaggregate the render phase(preprocessing) and - Disaggregated tokenization: Support use cases such as llm-d, Dynamo, and custom frontends that need to leverage vLLM's preprocessing logic without running the full inference engine. - Tokens-in / tokens-out engine: Make the engine a pure token-in / token-out service, decoupled from request preprocessing. +The dedicated `vllm launch render` server always exposes the `/render` and +`/derender` endpoints when `VLLM_ENABLE_SCALE_OUT_ENDPOINTS` is unset or set to +`1`. An explicit value of `0` conflicts with the renderer command and is +rejected at startup. + +Scale-out endpoints, including `/render`, `/derender`, and +`/inference/v1/generate`, are disabled by default on a standard inference +server. To expose them with `vllm serve`, opt in explicitly: + +```bash +VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 vllm serve +``` + ## API Reference - [Completions Render API](renderer.md) (`/v1/completions/render`) diff --git a/docs/usage/security.md b/docs/usage/security.md index 6e4822efc7..4314d39ce8 100644 --- a/docs/usage/security.md +++ b/docs/usage/security.md @@ -158,15 +158,16 @@ When `--api-key` is configured, the following `/v1` endpoints require Bearer tok - `/v1/models` - List available models - `/v1/chat/completions` - Chat completions - `/v1/chat/completions/batch` - Batch chat completions -- `/v1/chat/completions/render` - Render chat completion requests -- `/v1/chat/completions/derender` - Derender chat completion requests +- `/v1/chat/completions/render` - Render chat completion requests (available on `vllm serve` only when `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1`, or on `vllm launch render` unless explicitly disabled) +- `/v1/chat/completions/derender` - Derender chat completion requests (available on `vllm serve` only when `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1`, or on `vllm launch render` unless explicitly disabled) - `/v1/completions` - Text completions -- `/v1/completions/render` - Render completion requests -- `/v1/completions/derender` - Derender completion requests +- `/v1/completions/render` - Render completion requests (available on `vllm serve` only when `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1`, or on `vllm launch render` unless explicitly disabled) +- `/v1/completions/derender` - Derender completion requests (available on `vllm serve` only when `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1`, or on `vllm launch render` unless explicitly disabled) - `/v1/embeddings` - Generate embeddings - `/v1/audio/transcriptions` - Audio transcription - `/v1/audio/translations` - Audio translation - `/v1/messages` - Anthropic-compatible messages API +- `/v1/messages/render` - Render Anthropic-compatible messages (available on `vllm serve` only when `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1`, or on `vllm launch render` unless explicitly disabled) - `/v1/messages/count_tokens` - Count tokens for Anthropic messages - `/v1/responses` - Create a response - `/v1/responses/{response_id}` - Retrieve a response @@ -175,7 +176,8 @@ When `--api-key` is configured, the following `/v1` endpoints require Bearer tok - `/v1/rerank` - Reranking API - `/v1/load_lora_adapter` - Load a LoRA adapter (can alter model behavior; only available when `--enable-lora` is set and `VLLM_ALLOW_RUNTIME_LORA_UPDATING=True`) - `/v1/unload_lora_adapter` - Unload a LoRA adapter (can alter model behavior; only available when `--enable-lora` is set and `VLLM_ALLOW_RUNTIME_LORA_UPDATING=True`) -- `/inference/v1/generate` - Generate completions +- `/inference/v1/generate` - Generate completions (available when `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1`, or with `--tokens-only` unless explicitly disabled) +- `/cohere/v2/chat/render` - Render Cohere Chat v2 requests (requires both `VLLM_ENABLE_COHERE_API=1` and `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1`) - `/v2/embed` - Cohere Embed API - `/v2/rerank` - Cohere Rerank API @@ -203,11 +205,11 @@ The following endpoints **do not require authentication** even when `--api-key` - `/init_weight_transfer_engine` - Initialize weight transfer engine for RLHF - `/update_weights` - Update model weights (can alter model behavior) - `/get_world_size` - Get distributed world size -- `/abort_requests` - Abort in-flight requests (only when `--tokens-only` is also set) +- `/abort_requests` - Abort in-flight requests (available with `--tokens-only` unless `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=0`) **Utility endpoints:** -- `/tokenize` - Tokenize text +- `/tokenize` - Tokenize text (not disabled by `VLLM_ENABLE_SCALE_OUT_ENDPOINTS`) - `/detokenize` - Detokenize tokens - `/health` - Health check - `/ping` - SageMaker health check diff --git a/examples/scale_out/example_mm_serve.py b/examples/scale_out/example_mm_serve.py index 11d81236c5..157a32d3be 100644 --- a/examples/scale_out/example_mm_serve.py +++ b/examples/scale_out/example_mm_serve.py @@ -13,7 +13,8 @@ zero client-side transformation. Launch the server first: - vllm serve Qwen/Qwen3-VL-2B-Instruct \ + VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 \ + vllm serve Qwen/Qwen3-VL-2B-Instruct \ --dtype bfloat16 --max-model-len 4096 --enforce-eager Then run this script: diff --git a/examples/scale_out/token_generation_client.py b/examples/scale_out/token_generation_client.py index 836f54d504..922018f43d 100644 --- a/examples/scale_out/token_generation_client.py +++ b/examples/scale_out/token_generation_client.py @@ -1,5 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Call a token-in/token-out server started with: + +VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 vllm serve Qwen/Qwen3-0.6B +""" + import httpx from transformers import AutoTokenizer diff --git a/rust/src/server/src/routes.rs b/rust/src/server/src/routes.rs index 0cf23625e0..393a08450f 100644 --- a/rust/src/server/src/routes.rs +++ b/rust/src/server/src/routes.rs @@ -26,7 +26,7 @@ use axum::extract::DefaultBodyLimit; use axum::middleware::{from_fn, from_fn_with_state}; use axum::routing::{get, post}; use tower_http::trace::TraceLayer; -use tracing::warn; +use tracing::{info, warn}; use crate::DEFAULT_REQUEST_BODY_LIMIT_BYTES; use crate::middleware; @@ -45,12 +45,33 @@ fn runtime_lora_updating_enabled() -> bool { .is_some_and(|value| matches!(value.trim().to_lowercase().as_str(), "1" | "true")) } +fn parse_scale_out_endpoints_flag(value: Option<&str>) -> Result { + let Some(value) = value else { + return Ok(false); + }; + if value.is_empty() { + return Ok(false); + } + + match value.trim() { + "0" => Ok(false), + "1" => Ok(true), + _ => Err("VLLM_ENABLE_SCALE_OUT_ENDPOINTS must be 0 or 1".to_string()), + } +} + +fn scale_out_endpoints_enabled() -> bool { + let value = std::env::var("VLLM_ENABLE_SCALE_OUT_ENDPOINTS").ok(); + parse_scale_out_endpoints_flag(value.as_deref()).unwrap_or_else(|message| panic!("{message}")) +} + /// Build the minimal OpenAI-compatible router for one configured model. pub fn build_router(state: Arc) -> Router { build_router_with_options( state, server_dev_mode_enabled(), runtime_lora_updating_enabled(), + scale_out_endpoints_enabled(), ) } @@ -65,13 +86,22 @@ fn build_router_with_dev_mode_and_lora( dev_mode_enabled: bool, runtime_lora_updating_enabled: bool, ) -> Router { - build_router_with_options(state, dev_mode_enabled, runtime_lora_updating_enabled) + build_router_with_options(state, dev_mode_enabled, runtime_lora_updating_enabled, true) +} + +#[cfg(test)] +fn build_router_with_scale_out_endpoints( + state: Arc, + scale_out_endpoints_enabled: bool, +) -> Router { + build_router_with_options(state, false, false, scale_out_endpoints_enabled) } fn build_router_with_options( state: Arc, dev_mode_enabled: bool, runtime_lora_updating_enabled: bool, + scale_out_endpoints_enabled: bool, ) -> Router { let mut router = Router::new() // Health & monitoring @@ -85,8 +115,16 @@ fn build_router_with_options( .route("/v1/chat/completions", post(openai::chat_completions)) // vLLM specific endpoints .route("/tokenize", post(tokenize::tokenize)) - .route("/detokenize", post(tokenize::detokenize)) - .route("/inference/v1/generate", post(inference::generate)); + .route("/detokenize", post(tokenize::detokenize)); + + if scale_out_endpoints_enabled { + router = router.route("/inference/v1/generate", post(inference::generate)); + } else { + info!( + "scale-out endpoints are disabled; set \ + VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 to enable them" + ); + } if runtime_lora_updating_enabled { router = router diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 0912f78ab4..484eed35d6 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -54,6 +54,7 @@ use zeromq::{DealerSocket, PushSocket, ZmqMessage}; use super::{ build_router, build_router_with_dev_mode, build_router_with_dev_mode_and_lora, + build_router_with_scale_out_endpoints, parse_scale_out_endpoints_flag, render::build_router as build_render_router, }; use crate::config::{ApiServerOptions, CorsConfig}; @@ -716,6 +717,83 @@ async fn test_app_with_dev_mode(dev_mode_enabled: bool) -> axum::Router { ) } +#[test] +fn scale_out_flag_accepts_unset_empty_zero_one_and_whitespace() { + assert_eq!(parse_scale_out_endpoints_flag(None), Ok(false)); + assert_eq!(parse_scale_out_endpoints_flag(Some("")), Ok(false)); + assert_eq!(parse_scale_out_endpoints_flag(Some("0")), Ok(false)); + assert_eq!(parse_scale_out_endpoints_flag(Some("1")), Ok(true)); + assert_eq!(parse_scale_out_endpoints_flag(Some(" 1 ")), Ok(true)); +} + +#[test] +fn scale_out_flag_rejects_values_other_than_zero_or_one() { + for value in ["-1", "2", "01", "+1", "invalid", " "] { + assert_eq!( + parse_scale_out_endpoints_flag(Some(value)), + Err("VLLM_ENABLE_SCALE_OUT_ENDPOINTS must be 0 or 1".to_string()) + ); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn scale_out_generate_route_is_disabled_by_default() { + let (chat, _engine_task) = test_models_with_engine_outputs_and_backend( + b"engine-scale-out-disabled", + default_stream_output_specs(), + Arc::new(FakeChatBackend::new()), + ) + .await; + let mut app = build_router_with_scale_out_endpoints( + Arc::new(AppState::new(vec!["model".to_string()], chat)), + false, + ); + + let response = app + .call( + Request::builder() + .method("POST") + .uri("/inference/v1/generate") + .header("content-type", "application/json") + .body(Body::from("{}")) + .expect("build request"), + ) + .await + .expect("call app"); + + assert_eq!(response.status(), StatusCode::NOT_FOUND); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn scale_out_generate_route_can_be_enabled() { + let (chat, _engine_task) = test_models_with_engine_outputs_and_backend( + b"engine-scale-out-enabled", + default_stream_output_specs(), + Arc::new(FakeChatBackend::new()), + ) + .await; + let mut app = build_router_with_scale_out_endpoints( + Arc::new(AppState::new(vec!["model".to_string()], chat)), + true, + ); + + let response = app + .call( + Request::builder() + .method("POST") + .uri("/inference/v1/generate") + .header("content-type", "application/json") + .body(Body::from("{}")) + .expect("build request"), + ) + .await + .expect("call app"); + + assert_ne!(response.status(), StatusCode::NOT_FOUND); +} + /// Build a dev-mode router backed by a mock engine using a custom ready /// response, returning the router and the engine task handle so the engine /// stays alive for the duration of the test. @@ -916,10 +994,13 @@ async fn test_app_with_stream_output_specs( ) .await; ( - build_router(Arc::new(AppState::new( - vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()], - chat, - ))), + build_router_with_scale_out_endpoints( + Arc::new(AppState::new( + vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()], + chat, + )), + true, + ), engine_task, ) } @@ -1236,10 +1317,10 @@ async fn render_chat_response_can_be_submitted_to_generate_unchanged() { Arc::new(FakeChatBackend::new()), ) .await; - let mut generate_app = build_router(Arc::new(AppState::new( - vec!["render-model".to_string()], - chat, - ))); + let mut generate_app = build_router_with_scale_out_endpoints( + Arc::new(AppState::new(vec!["render-model".to_string()], chat)), + true, + ); let generate_response = generate_app .call( Request::builder() @@ -3986,10 +4067,13 @@ async fn non_stream_raw_generate_returns_token_output_envelope() { .await .expect("connect client"); let chat = ChatLlm::from_shared_backend(Llm::new(client), Arc::new(FakeChatBackend::new())); - let mut app = build_router(Arc::new(AppState::new( - vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()], - chat, - ))); + let mut app = build_router_with_scale_out_endpoints( + Arc::new(AppState::new( + vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()], + chat, + )), + true, + ); let response = app .call( @@ -4100,10 +4184,13 @@ async fn stream_raw_generate_returns_sse_chunks_and_usage() { .await .expect("connect client"); let chat = ChatLlm::from_shared_backend(Llm::new(client), Arc::new(FakeChatBackend::new())); - let mut app = build_router(Arc::new(AppState::new( - vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()], - chat, - ))); + let mut app = build_router_with_scale_out_endpoints( + Arc::new(AppState::new( + vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()], + chat, + )), + true, + ); let response = app .call( diff --git a/tests/entrypoints/cohere/test_api_router.py b/tests/entrypoints/cohere/test_api_router.py index 72fbc77cd3..a90e93b6b1 100644 --- a/tests/entrypoints/cohere/test_api_router.py +++ b/tests/entrypoints/cohere/test_api_router.py @@ -58,6 +58,7 @@ def _enable_cohere_api(monkeypatch): flag inside the test body. """ monkeypatch.setenv("VLLM_ENABLE_COHERE_API", "1") + monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "1") # ---------------------------------------------------------------------- @@ -402,11 +403,12 @@ class TestRenderEndpoint: assert "/cohere/v2/chat/render" in paths def test_route_not_registered_when_flag_unset(self, monkeypatch): - monkeypatch.delenv("VLLM_ENABLE_COHERE_API", raising=False) + monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False) app = FastAPI() attach_router(app) paths = [getattr(r, "path", None) for r in app.routes] assert "/cohere/v2/chat/render" not in paths + assert "/cohere/v2/chat" in paths def test_returns_generate_request_json(self): render_handler = _RenderHandler(_generate_request()) diff --git a/tests/entrypoints/cohere/test_chat_v2.py b/tests/entrypoints/cohere/test_chat_v2.py index b8e5cd4d50..36acecb622 100644 --- a/tests/entrypoints/cohere/test_chat_v2.py +++ b/tests/entrypoints/cohere/test_chat_v2.py @@ -71,6 +71,7 @@ def server(): env_dict = { "VLLM_CPU_KVCACHE_SPACE": "1", "VLLM_ENABLE_COHERE_API": "1", + "VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1", } with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server: diff --git a/tests/entrypoints/scale_out/derender/test_derender_parity.py b/tests/entrypoints/scale_out/derender/test_derender_parity.py index 8e51ff5d42..5b313464da 100644 --- a/tests/entrypoints/scale_out/derender/test_derender_parity.py +++ b/tests/entrypoints/scale_out/derender/test_derender_parity.py @@ -54,7 +54,11 @@ FORCE_WEATHER_TOOL = {"type": "function", "function": {"name": "get_weather"}} @pytest.fixture(scope="module") def server(): - with RemoteOpenAIServer(MODEL, ARGS) as remote_server: + with RemoteOpenAIServer( + MODEL, + ARGS, + env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"}, + ) as remote_server: yield remote_server diff --git a/tests/entrypoints/scale_out/render/test_render_multimodal.py b/tests/entrypoints/scale_out/render/test_render_multimodal.py index 459a965c04..70bbc95016 100644 --- a/tests/entrypoints/scale_out/render/test_render_multimodal.py +++ b/tests/entrypoints/scale_out/render/test_render_multimodal.py @@ -29,7 +29,7 @@ def vision_server(): "0", ] - env_overrides: dict[str, str] = {} + env_overrides = {"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"} with RemoteOpenAIServer( VISION_MODEL_NAME, diff --git a/tests/entrypoints/scale_out/test_factories.py b/tests/entrypoints/scale_out/test_factories.py new file mode 100644 index 0000000000..76b066ccf1 --- /dev/null +++ b/tests/entrypoints/scale_out/test_factories.py @@ -0,0 +1,89 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from argparse import Namespace + +import pytest +from fastapi import FastAPI + +from vllm.entrypoints.launchers.api_server.routers import register_api_routers +from vllm.tasks import SupportedTask + +RENDER_PATHS = { + "/v1/chat/completions/render", + "/v1/chat/completions/derender", + "/v1/completions/render", + "/v1/completions/derender", + "/v1/messages/render", +} +GENERATE_PATH = "/inference/v1/generate" +SCALE_OUT_PATHS = RENDER_PATHS | {GENERATE_PATH} + + +def registered_paths( + supported_tasks: tuple[SupportedTask, ...], *, tokens_only: bool = False +) -> set[str]: + app = FastAPI() + args = Namespace(tokens_only=tokens_only, enable_fault_tolerance=False) + app.state.args = args + register_api_routers(args, app, supported_tasks) + return {route.path for route in app.routes} + + +def test_scale_out_routes_are_disabled_by_default(monkeypatch): + monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False) + + paths = registered_paths(("generate",)) + + assert SCALE_OUT_PATHS.isdisjoint(paths) + + +def test_disabled_scale_out_routes_are_logged(monkeypatch, caplog): + monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False) + + with caplog.at_level("INFO", logger="vllm.entrypoints.scale_out.factories"): + registered_paths(("generate",)) + + assert any( + "VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1" in record.message + for record in caplog.records + ) + + +def test_scale_out_routes_can_be_enabled_for_generate_server(monkeypatch): + monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "1") + + paths = registered_paths(("generate",)) + + assert paths >= SCALE_OUT_PATHS + + +def test_render_routes_remain_enabled_for_render_server(monkeypatch): + monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False) + + paths = registered_paths(("render",)) + + assert paths >= RENDER_PATHS + assert GENERATE_PATH not in paths + + +def test_render_server_rejects_explicitly_disabled_scale_out_routes(monkeypatch): + monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "0") + + with pytest.raises(ValueError, match="VLLM_ENABLE_SCALE_OUT_ENDPOINTS=0"): + registered_paths(("render",)) + + +def test_tokens_only_mode_enables_generate_routes_when_flag_is_unset(monkeypatch): + monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False) + + paths = registered_paths(("generate",), tokens_only=True) + + assert paths >= {GENERATE_PATH, "/abort_requests"} + + +def test_tokens_only_mode_rejects_explicitly_disabled_scale_out_routes(monkeypatch): + monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "0") + + with pytest.raises(ValueError, match="--tokens-only"): + registered_paths(("generate",), tokens_only=True) diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_return_routed_experts.py b/tests/entrypoints/scale_out/token_in_token_out/test_return_routed_experts.py index f7fd179516..963e416e7e 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_return_routed_experts.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_return_routed_experts.py @@ -35,7 +35,11 @@ def server(): "--hf-overrides", '{"sliding_window": null}', ] - with RemoteOpenAIServer(MODEL_NAME, args) as remote_server: + with RemoteOpenAIServer( + MODEL_NAME, + args, + env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"}, + ) as remote_server: yield remote_server diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_serving_multimodal_tokens.py b/tests/entrypoints/scale_out/token_in_token_out/test_serving_multimodal_tokens.py index 60f5e9e420..9491623fd1 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_serving_multimodal_tokens.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_serving_multimodal_tokens.py @@ -39,7 +39,11 @@ def server(): "--no-enable-prefix-caching", ] - with RemoteOpenAIServer(MODEL_NAME, args) as remote_server: + with RemoteOpenAIServer( + MODEL_NAME, + args, + env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"}, + ) as remote_server: yield remote_server diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py b/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py index be33a182bc..dc682c6d9a 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py @@ -79,7 +79,11 @@ def server(request): else [str(extra_args)] ) - with RemoteOpenAIServer(MODEL_NAME, args) as remote_server: + with RemoteOpenAIServer( + MODEL_NAME, + args, + env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"}, + ) as remote_server: yield remote_server diff --git a/tests/test_envs.py b/tests/test_envs.py index 1ef40cde0c..aaa53d2a54 100644 --- a/tests/test_envs.py +++ b/tests/test_envs.py @@ -43,6 +43,41 @@ def test_api_key_is_not_compile_factor(monkeypatch: pytest.MonkeyPatch): assert "VLLM_API_KEY" not in envs.compile_factors() +def test_scale_out_endpoints_flag_is_runtime_only(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "1") + + assert envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS is True + assert "VLLM_ENABLE_SCALE_OUT_ENDPOINTS" not in envs.compile_factors() + + +def test_scale_out_endpoints_flag_distinguishes_unset_from_disabled( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False) + assert envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS is None + + monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "0") + assert envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS is False + + +def test_scale_out_endpoints_flag_treats_empty_as_unset( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "") + + assert envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS is None + + +@pytest.mark.parametrize("value", ["-1", "2", "01", "+1", "invalid", " "]) +def test_scale_out_endpoints_flag_rejects_values_other_than_zero_or_one( + monkeypatch: pytest.MonkeyPatch, value: str +): + monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", value) + + with pytest.raises(ValueError, match="must be 0 or 1"): + _ = envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS + + def test_p2p_side_channel_defaults_and_override(monkeypatch: pytest.MonkeyPatch): monkeypatch.delenv("VLLM_P2P_SIDE_CHANNEL_HOST", raising=False) monkeypatch.delenv("VLLM_P2P_SIDE_CHANNEL_PORT", raising=False) diff --git a/vllm/entrypoints/cohere/api_router.py b/vllm/entrypoints/cohere/api_router.py index f0f8d0c786..445964e6aa 100644 --- a/vllm/entrypoints/cohere/api_router.py +++ b/vllm/entrypoints/cohere/api_router.py @@ -17,9 +17,9 @@ log) and vLLM continues to boot normally. Even when the SDK is installed, :func:`attach_router` also requires ``VLLM_ENABLE_COHERE_API=1`` in the environment before it will expose -the routes. This keeps non-Cohere deployments that pull in the SDK for -unrelated reasons (e.g. test dependencies) from accidentally exposing -the api. +the chat route. The render route additionally requires +``VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1``. These gates keep unrelated +deployments from accidentally exposing the APIs. Note: the handlers must live at module scope (not inside ``attach_router``) so that FastAPI's ``typing.get_type_hints`` resolves @@ -71,6 +71,7 @@ if _SDK_AVAILABLE: from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest router = APIRouter() + render_router = APIRouter() def _serving(request: Request) -> CohereServingChatV2 | None: return getattr(request.app.state, "cohere_serving_chat_v2", None) @@ -154,7 +155,7 @@ if _SDK_AVAILABLE: case _: return StreamingResponse(content=result, media_type="text/event-stream") - @router.post( + @render_router.post( "/cohere/v2/chat/render", dependencies=[Depends(validate_json_request)], response_model=GenerateRequest, @@ -273,7 +274,7 @@ if _SDK_AVAILABLE: def attach_router(app: FastAPI) -> None: - """Register the ``/cohere/v2/chat`` routes on ``app``. + """Register the enabled ``/cohere/v2/chat`` routes on ``app``. No-op when either: @@ -305,4 +306,6 @@ def attach_router(app: FastAPI) -> None: ) return app.include_router(router) + if envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS: + app.include_router(render_router) app.add_middleware(CohereErrorEnvelopeMiddleware) diff --git a/vllm/entrypoints/scale_out/factories.py b/vllm/entrypoints/scale_out/factories.py index 341dad13a8..1af682c31e 100644 --- a/vllm/entrypoints/scale_out/factories.py +++ b/vllm/entrypoints/scale_out/factories.py @@ -5,7 +5,9 @@ from typing import TYPE_CHECKING from fastapi import FastAPI +from vllm import envs from vllm.engine.protocol import EngineClient +from vllm.logger import init_logger from vllm.tasks import SupportedTask if TYPE_CHECKING: @@ -15,6 +17,8 @@ if TYPE_CHECKING: else: RequestLogger = object +logger = init_logger(__name__) + def init_render_state( state: "State", @@ -62,6 +66,25 @@ def register_scale_out_api_routers( app: FastAPI, supported_tasks: tuple["SupportedTask", ...], ): + dedicated_mode = None + if "render" in supported_tasks: + dedicated_mode = "`vllm launch render`" + elif getattr(app.state.args, "tokens_only", False): + dedicated_mode = "`--tokens-only`" + + enabled = envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS + if dedicated_mode is not None and enabled is False: + raise ValueError( + "`VLLM_ENABLE_SCALE_OUT_ENDPOINTS=0` conflicts with " + f"{dedicated_mode}; unset it or set it to 1" + ) + if dedicated_mode is None and not enabled: + logger.info( + "Scale-out endpoints are disabled. Set " + "VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 to enable them." + ) + return + from .render.api_router import router as render_render app.include_router(render_render) diff --git a/vllm/envs.py b/vllm/envs.py index e33758d808..a3ef86d1b6 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -261,6 +261,7 @@ if TYPE_CHECKING: VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE: bool = True VLLM_ENABLE_RESPONSES_API_STORE: bool = False VLLM_ENABLE_COHERE_API: bool = False + VLLM_ENABLE_SCALE_OUT_ENDPOINTS: bool | None = None VLLM_HAS_FLASHINFER_CUBIN: bool = False VLLM_ROCM_FP8_MFMA_PAGE_ATTN: bool = False VLLM_ALLREDUCE_USE_SYMM_MEM: bool = True @@ -353,6 +354,15 @@ def maybe_convert_bool(value: str | None) -> bool | None: return bool(int(value)) +def maybe_convert_scale_out_endpoints(value: str | None) -> bool | None: + if value is None: + return None + normalized = value.strip() + if normalized not in ("0", "1"): + raise ValueError("VLLM_ENABLE_SCALE_OUT_ENDPOINTS must be 0 or 1") + return maybe_convert_bool(normalized) + + def maybe_convert_json_str_or_file(value: str | None) -> dict[str, Any] | None: if value is None: return None @@ -1864,6 +1874,12 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_ENABLE_COHERE_API": lambda: bool( int(os.getenv("VLLM_ENABLE_COHERE_API", "0")) ), + # If set to 1, expose the scale-out endpoints on `vllm serve`. + # The dedicated `vllm launch render` server exposes render and derender + # endpoints when this variable is unset or set to 1. + "VLLM_ENABLE_SCALE_OUT_ENDPOINTS": lambda: maybe_convert_scale_out_endpoints( + os.getenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS") or None + ), # If set, use the fp8 mfma in rocm paged attention. "VLLM_ROCM_FP8_MFMA_PAGE_ATTN": lambda: bool( int(os.getenv("VLLM_ROCM_FP8_MFMA_PAGE_ATTN", "0")) @@ -2291,6 +2307,7 @@ def compile_factors() -> dict[str, object]: "VLLM_CONFIG_ROOT", "LD_LIBRARY_PATH", "VLLM_SERVER_DEV_MODE", + "VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "VLLM_DP_MASTER_IP", "VLLM_DP_MASTER_PORT", "VLLM_NIXL_SIDE_CHANNEL_HOST",