From cd64c2dea9c72af333de7ec05293d54bbf1bd128 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Lucchesi?= Date: Mon, 7 Sep 2026 16:08:48 +0200 Subject: [PATCH] [Docs] Clarify admission control limits apply server-wide, not per DP rank (#55124) Signed-off-by: NickLucche --- docs/serving/data_parallel_deployment.md | 2 ++ vllm/config/scheduler.py | 13 ++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/serving/data_parallel_deployment.md b/docs/serving/data_parallel_deployment.md index 1f18b92f95..3b92fa06a5 100644 --- a/docs/serving/data_parallel_deployment.md +++ b/docs/serving/data_parallel_deployment.md @@ -26,6 +26,8 @@ vLLM supports "self-contained" data parallel deployments that expose a single AP It can be configured by simply including e.g. `--data-parallel-size=4` in the vllm serve command line arguments. This will require 4 GPUs. It can be combined with tensor parallel, for example `--data-parallel-size=4 --tensor-parallel-size=2`, which would require 8 GPUs. When sizing DP deployments, remember that `--max-num-seqs` applies per DP rank. +By contrast, the admission control limits `--max-num-queued-reqs` and `--max-num-queued-tokens` apply to the whole server, not per DP rank: each API server process counts in-flight requests and prefill backlog across all DP ranks it routes to. For example, with `--data-parallel-size=4 --max-num-seqs=256 --max-num-queued-reqs=256`, the deployment rejects new requests once 256 are in-flight in total, even though ranks could jointly run 1024. To keep ranks saturating, size the cap as roughly `data-parallel-size * max-num-seqs` plus the desired queue depth. + Running a single data parallel deployment across multiple nodes requires a different `vllm serve` to be run on each node, specifying which DP ranks should run on that node. In this case, there will still be a single HTTP entrypoint - the API server(s) will run only on one node, but it doesn't necessarily need to be co-located with the DP ranks. This will run DP=4, TP=2 on a single 8-GPU node: diff --git a/vllm/config/scheduler.py b/vllm/config/scheduler.py index 7ac0f8f837..8aa0ffa5dc 100644 --- a/vllm/config/scheduler.py +++ b/vllm/config/scheduler.py @@ -76,7 +76,13 @@ class SchedulerConfig: at the same time, or None for no limit. When the limit is reached, new requests are rejected with HTTP 503 so the client can retry on another instance. This bounds vLLM's otherwise unbounded request queue and is - primarily a coarse capacity valve.""" + primarily a coarse capacity valve. + + Unlike ``max_num_seqs``, which applies per data-parallel rank, this + limit is enforced in the API server process and counts in-flight + requests across all DP ranks it routes to. Size it as roughly + ``data_parallel_size * max_num_seqs`` plus the desired queue depth if + it should not bind before per-rank admission does.""" max_num_queued_tokens: int | None = Field(default=None, ge=0) """Maximum total prompt tokens of requests currently in the prefill @@ -89,6 +95,11 @@ class SchedulerConfig: prefill-decode setup this maps directly to the prefill pool's capacity. + Like ``max_num_queued_reqs``, this limit is enforced in the API + server process and covers the prefill backlog across all DP ranks it + routes to, so ``prefill_throughput`` in the formula above is the + aggregate throughput of the deployment. + Note: the count is conservative. A partially prefilled request still contributes its full ``prompt_len`` until it transitions out of the prefill phase, because the scheduler's per-iteration