[Frontend] Gate scale-out endpoints behind opt-in flag (#54579)
Signed-off-by: Francisco Javier Arceo <farceo@redhat.com>
This commit is contained in:
@@ -121,6 +121,13 @@ For further details on profiling vLLM, please refer to [this page](../../contrib
|
||||
|
||||
## Scale-Out APIs
|
||||
|
||||
Scale-out APIs are disabled by default on `vllm serve`. The environment
|
||||
variable accepts only `0` or `1`; set `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1` to
|
||||
register the endpoints below. The
|
||||
dedicated `vllm launch render` and `vllm serve --tokens-only` modes are explicit
|
||||
opt-ins and enable their required endpoints when the variable is unset; an
|
||||
explicit value of `0` is rejected for those modes.
|
||||
|
||||
### Tokens IN <> Tokens OUT APIs
|
||||
|
||||
- `/inference/v1/generate` - Generate completions
|
||||
|
||||
@@ -56,6 +56,14 @@ Oversized payloads are rejected with a `400` before any `tokenizer.decode()` or
|
||||
|
||||
The example below drives the full `render → generate → derender` round trip for a chat request against a GPU less render server (`/render`, `/derender`) and a token-in / token-out engine (`/inference/v1/generate`).
|
||||
|
||||
Launch the two servers first:
|
||||
|
||||
```bash
|
||||
vllm launch render meta-llama/Llama-3.2-1B-Instruct --port 8100
|
||||
VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 vllm serve \
|
||||
meta-llama/Llama-3.2-1B-Instruct --tokens-only --port 8200
|
||||
```
|
||||
|
||||
```python
|
||||
import httpx
|
||||
|
||||
|
||||
@@ -6,6 +6,19 @@ Our renderer API is designed to disaggregate the render phase(preprocessing) and
|
||||
- Disaggregated tokenization: Support use cases such as llm-d, Dynamo, and custom frontends that need to leverage vLLM's preprocessing logic without running the full inference engine.
|
||||
- Tokens-in / tokens-out engine: Make the engine a pure token-in / token-out service, decoupled from request preprocessing.
|
||||
|
||||
The dedicated `vllm launch render` server always exposes the `/render` and
|
||||
`/derender` endpoints when `VLLM_ENABLE_SCALE_OUT_ENDPOINTS` is unset or set to
|
||||
`1`. An explicit value of `0` conflicts with the renderer command and is
|
||||
rejected at startup.
|
||||
|
||||
Scale-out endpoints, including `/render`, `/derender`, and
|
||||
`/inference/v1/generate`, are disabled by default on a standard inference
|
||||
server. To expose them with `vllm serve`, opt in explicitly:
|
||||
|
||||
```bash
|
||||
VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 vllm serve <model>
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
- [Completions Render API](renderer.md) (`/v1/completions/render`)
|
||||
|
||||
@@ -158,15 +158,16 @@ When `--api-key` is configured, the following `/v1` endpoints require Bearer tok
|
||||
- `/v1/models` - List available models
|
||||
- `/v1/chat/completions` - Chat completions
|
||||
- `/v1/chat/completions/batch` - Batch chat completions
|
||||
- `/v1/chat/completions/render` - Render chat completion requests
|
||||
- `/v1/chat/completions/derender` - Derender chat completion requests
|
||||
- `/v1/chat/completions/render` - Render chat completion requests (available on `vllm serve` only when `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1`, or on `vllm launch render` unless explicitly disabled)
|
||||
- `/v1/chat/completions/derender` - Derender chat completion requests (available on `vllm serve` only when `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1`, or on `vllm launch render` unless explicitly disabled)
|
||||
- `/v1/completions` - Text completions
|
||||
- `/v1/completions/render` - Render completion requests
|
||||
- `/v1/completions/derender` - Derender completion requests
|
||||
- `/v1/completions/render` - Render completion requests (available on `vllm serve` only when `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1`, or on `vllm launch render` unless explicitly disabled)
|
||||
- `/v1/completions/derender` - Derender completion requests (available on `vllm serve` only when `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1`, or on `vllm launch render` unless explicitly disabled)
|
||||
- `/v1/embeddings` - Generate embeddings
|
||||
- `/v1/audio/transcriptions` - Audio transcription
|
||||
- `/v1/audio/translations` - Audio translation
|
||||
- `/v1/messages` - Anthropic-compatible messages API
|
||||
- `/v1/messages/render` - Render Anthropic-compatible messages (available on `vllm serve` only when `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1`, or on `vllm launch render` unless explicitly disabled)
|
||||
- `/v1/messages/count_tokens` - Count tokens for Anthropic messages
|
||||
- `/v1/responses` - Create a response
|
||||
- `/v1/responses/{response_id}` - Retrieve a response
|
||||
@@ -175,7 +176,8 @@ When `--api-key` is configured, the following `/v1` endpoints require Bearer tok
|
||||
- `/v1/rerank` - Reranking API
|
||||
- `/v1/load_lora_adapter` - Load a LoRA adapter (can alter model behavior; only available when `--enable-lora` is set and `VLLM_ALLOW_RUNTIME_LORA_UPDATING=True`)
|
||||
- `/v1/unload_lora_adapter` - Unload a LoRA adapter (can alter model behavior; only available when `--enable-lora` is set and `VLLM_ALLOW_RUNTIME_LORA_UPDATING=True`)
|
||||
- `/inference/v1/generate` - Generate completions
|
||||
- `/inference/v1/generate` - Generate completions (available when `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1`, or with `--tokens-only` unless explicitly disabled)
|
||||
- `/cohere/v2/chat/render` - Render Cohere Chat v2 requests (requires both `VLLM_ENABLE_COHERE_API=1` and `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1`)
|
||||
- `/v2/embed` - Cohere Embed API
|
||||
- `/v2/rerank` - Cohere Rerank API
|
||||
|
||||
@@ -203,11 +205,11 @@ The following endpoints **do not require authentication** even when `--api-key`
|
||||
- `/init_weight_transfer_engine` - Initialize weight transfer engine for RLHF
|
||||
- `/update_weights` - Update model weights (can alter model behavior)
|
||||
- `/get_world_size` - Get distributed world size
|
||||
- `/abort_requests` - Abort in-flight requests (only when `--tokens-only` is also set)
|
||||
- `/abort_requests` - Abort in-flight requests (available with `--tokens-only` unless `VLLM_ENABLE_SCALE_OUT_ENDPOINTS=0`)
|
||||
|
||||
**Utility endpoints:**
|
||||
|
||||
- `/tokenize` - Tokenize text
|
||||
- `/tokenize` - Tokenize text (not disabled by `VLLM_ENABLE_SCALE_OUT_ENDPOINTS`)
|
||||
- `/detokenize` - Detokenize tokens
|
||||
- `/health` - Health check
|
||||
- `/ping` - SageMaker health check
|
||||
|
||||
@@ -13,7 +13,8 @@ zero client-side transformation.
|
||||
|
||||
Launch the server first:
|
||||
|
||||
vllm serve Qwen/Qwen3-VL-2B-Instruct \
|
||||
VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 \
|
||||
vllm serve Qwen/Qwen3-VL-2B-Instruct \
|
||||
--dtype bfloat16 --max-model-len 4096 --enforce-eager
|
||||
|
||||
Then run this script:
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Call a token-in/token-out server started with:
|
||||
|
||||
VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 vllm serve Qwen/Qwen3-0.6B
|
||||
"""
|
||||
|
||||
import httpx
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ use axum::extract::DefaultBodyLimit;
|
||||
use axum::middleware::{from_fn, from_fn_with_state};
|
||||
use axum::routing::{get, post};
|
||||
use tower_http::trace::TraceLayer;
|
||||
use tracing::warn;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::DEFAULT_REQUEST_BODY_LIMIT_BYTES;
|
||||
use crate::middleware;
|
||||
@@ -45,12 +45,33 @@ fn runtime_lora_updating_enabled() -> bool {
|
||||
.is_some_and(|value| matches!(value.trim().to_lowercase().as_str(), "1" | "true"))
|
||||
}
|
||||
|
||||
fn parse_scale_out_endpoints_flag(value: Option<&str>) -> Result<bool, String> {
|
||||
let Some(value) = value else {
|
||||
return Ok(false);
|
||||
};
|
||||
if value.is_empty() {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
match value.trim() {
|
||||
"0" => Ok(false),
|
||||
"1" => Ok(true),
|
||||
_ => Err("VLLM_ENABLE_SCALE_OUT_ENDPOINTS must be 0 or 1".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
fn scale_out_endpoints_enabled() -> bool {
|
||||
let value = std::env::var("VLLM_ENABLE_SCALE_OUT_ENDPOINTS").ok();
|
||||
parse_scale_out_endpoints_flag(value.as_deref()).unwrap_or_else(|message| panic!("{message}"))
|
||||
}
|
||||
|
||||
/// Build the minimal OpenAI-compatible router for one configured model.
|
||||
pub fn build_router(state: Arc<AppState>) -> Router {
|
||||
build_router_with_options(
|
||||
state,
|
||||
server_dev_mode_enabled(),
|
||||
runtime_lora_updating_enabled(),
|
||||
scale_out_endpoints_enabled(),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -65,13 +86,22 @@ fn build_router_with_dev_mode_and_lora(
|
||||
dev_mode_enabled: bool,
|
||||
runtime_lora_updating_enabled: bool,
|
||||
) -> Router {
|
||||
build_router_with_options(state, dev_mode_enabled, runtime_lora_updating_enabled)
|
||||
build_router_with_options(state, dev_mode_enabled, runtime_lora_updating_enabled, true)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn build_router_with_scale_out_endpoints(
|
||||
state: Arc<AppState>,
|
||||
scale_out_endpoints_enabled: bool,
|
||||
) -> Router {
|
||||
build_router_with_options(state, false, false, scale_out_endpoints_enabled)
|
||||
}
|
||||
|
||||
fn build_router_with_options(
|
||||
state: Arc<AppState>,
|
||||
dev_mode_enabled: bool,
|
||||
runtime_lora_updating_enabled: bool,
|
||||
scale_out_endpoints_enabled: bool,
|
||||
) -> Router {
|
||||
let mut router = Router::new()
|
||||
// Health & monitoring
|
||||
@@ -85,8 +115,16 @@ fn build_router_with_options(
|
||||
.route("/v1/chat/completions", post(openai::chat_completions))
|
||||
// vLLM specific endpoints
|
||||
.route("/tokenize", post(tokenize::tokenize))
|
||||
.route("/detokenize", post(tokenize::detokenize))
|
||||
.route("/inference/v1/generate", post(inference::generate));
|
||||
.route("/detokenize", post(tokenize::detokenize));
|
||||
|
||||
if scale_out_endpoints_enabled {
|
||||
router = router.route("/inference/v1/generate", post(inference::generate));
|
||||
} else {
|
||||
info!(
|
||||
"scale-out endpoints are disabled; set \
|
||||
VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 to enable them"
|
||||
);
|
||||
}
|
||||
|
||||
if runtime_lora_updating_enabled {
|
||||
router = router
|
||||
|
||||
@@ -54,6 +54,7 @@ use zeromq::{DealerSocket, PushSocket, ZmqMessage};
|
||||
|
||||
use super::{
|
||||
build_router, build_router_with_dev_mode, build_router_with_dev_mode_and_lora,
|
||||
build_router_with_scale_out_endpoints, parse_scale_out_endpoints_flag,
|
||||
render::build_router as build_render_router,
|
||||
};
|
||||
use crate::config::{ApiServerOptions, CorsConfig};
|
||||
@@ -716,6 +717,83 @@ async fn test_app_with_dev_mode(dev_mode_enabled: bool) -> axum::Router {
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scale_out_flag_accepts_unset_empty_zero_one_and_whitespace() {
|
||||
assert_eq!(parse_scale_out_endpoints_flag(None), Ok(false));
|
||||
assert_eq!(parse_scale_out_endpoints_flag(Some("")), Ok(false));
|
||||
assert_eq!(parse_scale_out_endpoints_flag(Some("0")), Ok(false));
|
||||
assert_eq!(parse_scale_out_endpoints_flag(Some("1")), Ok(true));
|
||||
assert_eq!(parse_scale_out_endpoints_flag(Some(" 1 ")), Ok(true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scale_out_flag_rejects_values_other_than_zero_or_one() {
|
||||
for value in ["-1", "2", "01", "+1", "invalid", " "] {
|
||||
assert_eq!(
|
||||
parse_scale_out_endpoints_flag(Some(value)),
|
||||
Err("VLLM_ENABLE_SCALE_OUT_ENDPOINTS must be 0 or 1".to_string())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn scale_out_generate_route_is_disabled_by_default() {
|
||||
let (chat, _engine_task) = test_models_with_engine_outputs_and_backend(
|
||||
b"engine-scale-out-disabled",
|
||||
default_stream_output_specs(),
|
||||
Arc::new(FakeChatBackend::new()),
|
||||
)
|
||||
.await;
|
||||
let mut app = build_router_with_scale_out_endpoints(
|
||||
Arc::new(AppState::new(vec!["model".to_string()], chat)),
|
||||
false,
|
||||
);
|
||||
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/inference/v1/generate")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from("{}"))
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_eq!(response.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn scale_out_generate_route_can_be_enabled() {
|
||||
let (chat, _engine_task) = test_models_with_engine_outputs_and_backend(
|
||||
b"engine-scale-out-enabled",
|
||||
default_stream_output_specs(),
|
||||
Arc::new(FakeChatBackend::new()),
|
||||
)
|
||||
.await;
|
||||
let mut app = build_router_with_scale_out_endpoints(
|
||||
Arc::new(AppState::new(vec!["model".to_string()], chat)),
|
||||
true,
|
||||
);
|
||||
|
||||
let response = app
|
||||
.call(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/inference/v1/generate")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from("{}"))
|
||||
.expect("build request"),
|
||||
)
|
||||
.await
|
||||
.expect("call app");
|
||||
|
||||
assert_ne!(response.status(), StatusCode::NOT_FOUND);
|
||||
}
|
||||
|
||||
/// Build a dev-mode router backed by a mock engine using a custom ready
|
||||
/// response, returning the router and the engine task handle so the engine
|
||||
/// stays alive for the duration of the test.
|
||||
@@ -916,10 +994,13 @@ async fn test_app_with_stream_output_specs(
|
||||
)
|
||||
.await;
|
||||
(
|
||||
build_router(Arc::new(AppState::new(
|
||||
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
|
||||
chat,
|
||||
))),
|
||||
build_router_with_scale_out_endpoints(
|
||||
Arc::new(AppState::new(
|
||||
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
|
||||
chat,
|
||||
)),
|
||||
true,
|
||||
),
|
||||
engine_task,
|
||||
)
|
||||
}
|
||||
@@ -1236,10 +1317,10 @@ async fn render_chat_response_can_be_submitted_to_generate_unchanged() {
|
||||
Arc::new(FakeChatBackend::new()),
|
||||
)
|
||||
.await;
|
||||
let mut generate_app = build_router(Arc::new(AppState::new(
|
||||
vec!["render-model".to_string()],
|
||||
chat,
|
||||
)));
|
||||
let mut generate_app = build_router_with_scale_out_endpoints(
|
||||
Arc::new(AppState::new(vec!["render-model".to_string()], chat)),
|
||||
true,
|
||||
);
|
||||
let generate_response = generate_app
|
||||
.call(
|
||||
Request::builder()
|
||||
@@ -3986,10 +4067,13 @@ async fn non_stream_raw_generate_returns_token_output_envelope() {
|
||||
.await
|
||||
.expect("connect client");
|
||||
let chat = ChatLlm::from_shared_backend(Llm::new(client), Arc::new(FakeChatBackend::new()));
|
||||
let mut app = build_router(Arc::new(AppState::new(
|
||||
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
|
||||
chat,
|
||||
)));
|
||||
let mut app = build_router_with_scale_out_endpoints(
|
||||
Arc::new(AppState::new(
|
||||
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
|
||||
chat,
|
||||
)),
|
||||
true,
|
||||
);
|
||||
|
||||
let response = app
|
||||
.call(
|
||||
@@ -4100,10 +4184,13 @@ async fn stream_raw_generate_returns_sse_chunks_and_usage() {
|
||||
.await
|
||||
.expect("connect client");
|
||||
let chat = ChatLlm::from_shared_backend(Llm::new(client), Arc::new(FakeChatBackend::new()));
|
||||
let mut app = build_router(Arc::new(AppState::new(
|
||||
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
|
||||
chat,
|
||||
)));
|
||||
let mut app = build_router_with_scale_out_endpoints(
|
||||
Arc::new(AppState::new(
|
||||
vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()],
|
||||
chat,
|
||||
)),
|
||||
true,
|
||||
);
|
||||
|
||||
let response = app
|
||||
.call(
|
||||
|
||||
@@ -58,6 +58,7 @@ def _enable_cohere_api(monkeypatch):
|
||||
flag inside the test body.
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_ENABLE_COHERE_API", "1")
|
||||
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "1")
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
@@ -402,11 +403,12 @@ class TestRenderEndpoint:
|
||||
assert "/cohere/v2/chat/render" in paths
|
||||
|
||||
def test_route_not_registered_when_flag_unset(self, monkeypatch):
|
||||
monkeypatch.delenv("VLLM_ENABLE_COHERE_API", raising=False)
|
||||
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
|
||||
app = FastAPI()
|
||||
attach_router(app)
|
||||
paths = [getattr(r, "path", None) for r in app.routes]
|
||||
assert "/cohere/v2/chat/render" not in paths
|
||||
assert "/cohere/v2/chat" in paths
|
||||
|
||||
def test_returns_generate_request_json(self):
|
||||
render_handler = _RenderHandler(_generate_request())
|
||||
|
||||
@@ -71,6 +71,7 @@ def server():
|
||||
env_dict = {
|
||||
"VLLM_CPU_KVCACHE_SPACE": "1",
|
||||
"VLLM_ENABLE_COHERE_API": "1",
|
||||
"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1",
|
||||
}
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server:
|
||||
|
||||
@@ -54,7 +54,11 @@ FORCE_WEATHER_TOOL = {"type": "function", "function": {"name": "get_weather"}}
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
with RemoteOpenAIServer(MODEL, ARGS) as remote_server:
|
||||
with RemoteOpenAIServer(
|
||||
MODEL,
|
||||
ARGS,
|
||||
env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"},
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
|
||||
@@ -29,7 +29,7 @@ def vision_server():
|
||||
"0",
|
||||
]
|
||||
|
||||
env_overrides: dict[str, str] = {}
|
||||
env_overrides = {"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"}
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
VISION_MODEL_NAME,
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from argparse import Namespace
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
|
||||
from vllm.entrypoints.launchers.api_server.routers import register_api_routers
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
RENDER_PATHS = {
|
||||
"/v1/chat/completions/render",
|
||||
"/v1/chat/completions/derender",
|
||||
"/v1/completions/render",
|
||||
"/v1/completions/derender",
|
||||
"/v1/messages/render",
|
||||
}
|
||||
GENERATE_PATH = "/inference/v1/generate"
|
||||
SCALE_OUT_PATHS = RENDER_PATHS | {GENERATE_PATH}
|
||||
|
||||
|
||||
def registered_paths(
|
||||
supported_tasks: tuple[SupportedTask, ...], *, tokens_only: bool = False
|
||||
) -> set[str]:
|
||||
app = FastAPI()
|
||||
args = Namespace(tokens_only=tokens_only, enable_fault_tolerance=False)
|
||||
app.state.args = args
|
||||
register_api_routers(args, app, supported_tasks)
|
||||
return {route.path for route in app.routes}
|
||||
|
||||
|
||||
def test_scale_out_routes_are_disabled_by_default(monkeypatch):
|
||||
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
|
||||
|
||||
paths = registered_paths(("generate",))
|
||||
|
||||
assert SCALE_OUT_PATHS.isdisjoint(paths)
|
||||
|
||||
|
||||
def test_disabled_scale_out_routes_are_logged(monkeypatch, caplog):
|
||||
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
|
||||
|
||||
with caplog.at_level("INFO", logger="vllm.entrypoints.scale_out.factories"):
|
||||
registered_paths(("generate",))
|
||||
|
||||
assert any(
|
||||
"VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1" in record.message
|
||||
for record in caplog.records
|
||||
)
|
||||
|
||||
|
||||
def test_scale_out_routes_can_be_enabled_for_generate_server(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "1")
|
||||
|
||||
paths = registered_paths(("generate",))
|
||||
|
||||
assert paths >= SCALE_OUT_PATHS
|
||||
|
||||
|
||||
def test_render_routes_remain_enabled_for_render_server(monkeypatch):
|
||||
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
|
||||
|
||||
paths = registered_paths(("render",))
|
||||
|
||||
assert paths >= RENDER_PATHS
|
||||
assert GENERATE_PATH not in paths
|
||||
|
||||
|
||||
def test_render_server_rejects_explicitly_disabled_scale_out_routes(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "0")
|
||||
|
||||
with pytest.raises(ValueError, match="VLLM_ENABLE_SCALE_OUT_ENDPOINTS=0"):
|
||||
registered_paths(("render",))
|
||||
|
||||
|
||||
def test_tokens_only_mode_enables_generate_routes_when_flag_is_unset(monkeypatch):
|
||||
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
|
||||
|
||||
paths = registered_paths(("generate",), tokens_only=True)
|
||||
|
||||
assert paths >= {GENERATE_PATH, "/abort_requests"}
|
||||
|
||||
|
||||
def test_tokens_only_mode_rejects_explicitly_disabled_scale_out_routes(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "0")
|
||||
|
||||
with pytest.raises(ValueError, match="--tokens-only"):
|
||||
registered_paths(("generate",), tokens_only=True)
|
||||
@@ -35,7 +35,11 @@ def server():
|
||||
"--hf-overrides",
|
||||
'{"sliding_window": null}',
|
||||
]
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME,
|
||||
args,
|
||||
env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"},
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
|
||||
@@ -39,7 +39,11 @@ def server():
|
||||
"--no-enable-prefix-caching",
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME,
|
||||
args,
|
||||
env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"},
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
|
||||
@@ -79,7 +79,11 @@ def server(request):
|
||||
else [str(extra_args)]
|
||||
)
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
with RemoteOpenAIServer(
|
||||
MODEL_NAME,
|
||||
args,
|
||||
env_dict={"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": "1"},
|
||||
) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
|
||||
@@ -43,6 +43,41 @@ def test_api_key_is_not_compile_factor(monkeypatch: pytest.MonkeyPatch):
|
||||
assert "VLLM_API_KEY" not in envs.compile_factors()
|
||||
|
||||
|
||||
def test_scale_out_endpoints_flag_is_runtime_only(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "1")
|
||||
|
||||
assert envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS is True
|
||||
assert "VLLM_ENABLE_SCALE_OUT_ENDPOINTS" not in envs.compile_factors()
|
||||
|
||||
|
||||
def test_scale_out_endpoints_flag_distinguishes_unset_from_disabled(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.delenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", raising=False)
|
||||
assert envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS is None
|
||||
|
||||
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "0")
|
||||
assert envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS is False
|
||||
|
||||
|
||||
def test_scale_out_endpoints_flag_treats_empty_as_unset(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", "")
|
||||
|
||||
assert envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["-1", "2", "01", "+1", "invalid", " "])
|
||||
def test_scale_out_endpoints_flag_rejects_values_other_than_zero_or_one(
|
||||
monkeypatch: pytest.MonkeyPatch, value: str
|
||||
):
|
||||
monkeypatch.setenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS", value)
|
||||
|
||||
with pytest.raises(ValueError, match="must be 0 or 1"):
|
||||
_ = envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS
|
||||
|
||||
|
||||
def test_p2p_side_channel_defaults_and_override(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.delenv("VLLM_P2P_SIDE_CHANNEL_HOST", raising=False)
|
||||
monkeypatch.delenv("VLLM_P2P_SIDE_CHANNEL_PORT", raising=False)
|
||||
|
||||
@@ -17,9 +17,9 @@ log) and vLLM continues to boot normally.
|
||||
|
||||
Even when the SDK is installed, :func:`attach_router` also requires
|
||||
``VLLM_ENABLE_COHERE_API=1`` in the environment before it will expose
|
||||
the routes. This keeps non-Cohere deployments that pull in the SDK for
|
||||
unrelated reasons (e.g. test dependencies) from accidentally exposing
|
||||
the api.
|
||||
the chat route. The render route additionally requires
|
||||
``VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1``. These gates keep unrelated
|
||||
deployments from accidentally exposing the APIs.
|
||||
|
||||
Note: the handlers must live at module scope (not inside
|
||||
``attach_router``) so that FastAPI's ``typing.get_type_hints`` resolves
|
||||
@@ -71,6 +71,7 @@ if _SDK_AVAILABLE:
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
|
||||
|
||||
router = APIRouter()
|
||||
render_router = APIRouter()
|
||||
|
||||
def _serving(request: Request) -> CohereServingChatV2 | None:
|
||||
return getattr(request.app.state, "cohere_serving_chat_v2", None)
|
||||
@@ -154,7 +155,7 @@ if _SDK_AVAILABLE:
|
||||
case _:
|
||||
return StreamingResponse(content=result, media_type="text/event-stream")
|
||||
|
||||
@router.post(
|
||||
@render_router.post(
|
||||
"/cohere/v2/chat/render",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
response_model=GenerateRequest,
|
||||
@@ -273,7 +274,7 @@ if _SDK_AVAILABLE:
|
||||
|
||||
|
||||
def attach_router(app: FastAPI) -> None:
|
||||
"""Register the ``/cohere/v2/chat`` routes on ``app``.
|
||||
"""Register the enabled ``/cohere/v2/chat`` routes on ``app``.
|
||||
|
||||
No-op when either:
|
||||
|
||||
@@ -305,4 +306,6 @@ def attach_router(app: FastAPI) -> None:
|
||||
)
|
||||
return
|
||||
app.include_router(router)
|
||||
if envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS:
|
||||
app.include_router(render_router)
|
||||
app.add_middleware(CohereErrorEnvelopeMiddleware)
|
||||
|
||||
@@ -5,7 +5,9 @@ from typing import TYPE_CHECKING
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
from vllm import envs
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -15,6 +17,8 @@ if TYPE_CHECKING:
|
||||
else:
|
||||
RequestLogger = object
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def init_render_state(
|
||||
state: "State",
|
||||
@@ -62,6 +66,25 @@ def register_scale_out_api_routers(
|
||||
app: FastAPI,
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
):
|
||||
dedicated_mode = None
|
||||
if "render" in supported_tasks:
|
||||
dedicated_mode = "`vllm launch render`"
|
||||
elif getattr(app.state.args, "tokens_only", False):
|
||||
dedicated_mode = "`--tokens-only`"
|
||||
|
||||
enabled = envs.VLLM_ENABLE_SCALE_OUT_ENDPOINTS
|
||||
if dedicated_mode is not None and enabled is False:
|
||||
raise ValueError(
|
||||
"`VLLM_ENABLE_SCALE_OUT_ENDPOINTS=0` conflicts with "
|
||||
f"{dedicated_mode}; unset it or set it to 1"
|
||||
)
|
||||
if dedicated_mode is None and not enabled:
|
||||
logger.info(
|
||||
"Scale-out endpoints are disabled. Set "
|
||||
"VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 to enable them."
|
||||
)
|
||||
return
|
||||
|
||||
from .render.api_router import router as render_render
|
||||
|
||||
app.include_router(render_render)
|
||||
|
||||
@@ -261,6 +261,7 @@ if TYPE_CHECKING:
|
||||
VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE: bool = True
|
||||
VLLM_ENABLE_RESPONSES_API_STORE: bool = False
|
||||
VLLM_ENABLE_COHERE_API: bool = False
|
||||
VLLM_ENABLE_SCALE_OUT_ENDPOINTS: bool | None = None
|
||||
VLLM_HAS_FLASHINFER_CUBIN: bool = False
|
||||
VLLM_ROCM_FP8_MFMA_PAGE_ATTN: bool = False
|
||||
VLLM_ALLREDUCE_USE_SYMM_MEM: bool = True
|
||||
@@ -353,6 +354,15 @@ def maybe_convert_bool(value: str | None) -> bool | None:
|
||||
return bool(int(value))
|
||||
|
||||
|
||||
def maybe_convert_scale_out_endpoints(value: str | None) -> bool | None:
|
||||
if value is None:
|
||||
return None
|
||||
normalized = value.strip()
|
||||
if normalized not in ("0", "1"):
|
||||
raise ValueError("VLLM_ENABLE_SCALE_OUT_ENDPOINTS must be 0 or 1")
|
||||
return maybe_convert_bool(normalized)
|
||||
|
||||
|
||||
def maybe_convert_json_str_or_file(value: str | None) -> dict[str, Any] | None:
|
||||
if value is None:
|
||||
return None
|
||||
@@ -1864,6 +1874,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_ENABLE_COHERE_API": lambda: bool(
|
||||
int(os.getenv("VLLM_ENABLE_COHERE_API", "0"))
|
||||
),
|
||||
# If set to 1, expose the scale-out endpoints on `vllm serve`.
|
||||
# The dedicated `vllm launch render` server exposes render and derender
|
||||
# endpoints when this variable is unset or set to 1.
|
||||
"VLLM_ENABLE_SCALE_OUT_ENDPOINTS": lambda: maybe_convert_scale_out_endpoints(
|
||||
os.getenv("VLLM_ENABLE_SCALE_OUT_ENDPOINTS") or None
|
||||
),
|
||||
# If set, use the fp8 mfma in rocm paged attention.
|
||||
"VLLM_ROCM_FP8_MFMA_PAGE_ATTN": lambda: bool(
|
||||
int(os.getenv("VLLM_ROCM_FP8_MFMA_PAGE_ATTN", "0"))
|
||||
@@ -2291,6 +2307,7 @@ def compile_factors() -> dict[str, object]:
|
||||
"VLLM_CONFIG_ROOT",
|
||||
"LD_LIBRARY_PATH",
|
||||
"VLLM_SERVER_DEV_MODE",
|
||||
"VLLM_ENABLE_SCALE_OUT_ENDPOINTS",
|
||||
"VLLM_DP_MASTER_IP",
|
||||
"VLLM_DP_MASTER_PORT",
|
||||
"VLLM_NIXL_SIDE_CHANNEL_HOST",
|
||||
|
||||
Reference in New Issue
Block a user