[Frontend] DP Supervisor (#40841)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
Signed-off-by: Robert Shaw <robertgshaw2@gmail.com>
Signed-off-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
Co-authored-by: robertgshaw2-redhat <robertgshaw2@gmail.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Wentao Ye
2026-05-21 20:33:16 -07:00
committed by GitHub
co-authored by robertgshaw2-redhat Robert Shaw Nick Hill
parent 60af5c16ee
commit 0ddd7dd656
7 changed files with 1168 additions and 17 deletions
+30 -14
View File
@@ -12,6 +12,9 @@ import vllm.envs as envs
from vllm.entrypoints.cli.types import CLISubcommand
from vllm.entrypoints.openai.api_server import run_server, setup_server
from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
from vllm.entrypoints.openai.dp_supervisor import (
run_dp_supervisor,
)
from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG
from vllm.logger import init_logger
from vllm.usage.usage_lib import UsageContext
@@ -66,31 +69,41 @@ class ServeSubcommand(CLISubcommand):
args.api_server_count = 0
# Detect LB mode for defaulting api_server_count.
# Multi-port: --data-parallel-multi-port-external-lb
# External LB: --data-parallel-external-lb or --data-parallel-rank
# Hybrid LB: --data-parallel-hybrid-lb or --data-parallel-start-rank
is_external_lb = (
args.data_parallel_external_lb or args.data_parallel_rank is not None
)
is_hybrid_lb = (
args.data_parallel_hybrid_lb or args.data_parallel_start_rank is not None
)
if is_external_lb and is_hybrid_lb:
# If --data_parallel_multi_port_external_lb and --data_parallel_hybrid_lb
# are unset, default to hybrid if --data-parallel-start-rank is set
is_hybrid_lb = is_multi_port = False
if (
not args.data_parallel_hybrid_lb
and not args.data_parallel_multi_port_external_lb
):
is_hybrid_lb = args.data_parallel_start_rank is not None
else:
is_hybrid_lb = args.data_parallel_hybrid_lb
is_multi_port = args.data_parallel_multi_port_external_lb
if sum([is_multi_port, is_external_lb, is_hybrid_lb]) > 1:
raise ValueError(
"Cannot use both external and hybrid data parallel load "
"balancing modes. External LB is enabled via "
"--data-parallel-external-lb or --data-parallel-rank. "
"Hybrid LB is enabled via --data-parallel-hybrid-lb or "
"--data-parallel-start-rank. Use one mode or the other."
"Cannot use more than one data parallel load balancing mode. "
"Choose one of: --data-parallel-multi-port-external-lb, "
"--data-parallel-external-lb (or --data-parallel-rank), "
"--data-parallel-hybrid-lb (or --data-parallel-start-rank)."
)
# Default api_server_count if not explicitly set.
# - Rust frontend: Use 1 (not applicable as it's multithreaded)
# - External LB: Leave as 1 (external LB handles distribution)
# - Multi-port: 1 (supervisor spawns one server per local DP rank)
# - Rust frontend: 1 (not applicable as it's multithreaded)
# - External LB: 1 (external LB handles distribution)
# - Hybrid LB: Use local DP size (internal LB for local ranks only)
# - Internal LB: Use full DP size
if args.api_server_count is None:
if is_external_lb or envs.VLLM_RUST_FRONTEND_PATH:
if is_multi_port or is_external_lb or envs.VLLM_RUST_FRONTEND_PATH:
args.api_server_count = 1
elif is_hybrid_lb:
args.api_server_count = args.data_parallel_size_local or 1
@@ -109,7 +122,8 @@ class ServeSubcommand(CLISubcommand):
)
elif envs.VLLM_RUST_FRONTEND_PATH and args.api_server_count > 1:
logger.warning(
"Ignoring --api-server-count=%d when using rust front-end process"
"Ignoring --api-server-count=%d when using rust front-end process",
args.api_server_count,
)
args.api_server_count = 1
@@ -122,7 +136,9 @@ class ServeSubcommand(CLISubcommand):
)
args.api_server_count = 1
if args.api_server_count < 1:
if is_multi_port:
run_dp_supervisor(args)
elif args.api_server_count < 1:
run_headless(args)
elif args.api_server_count > 1 or envs.VLLM_RUST_FRONTEND_PATH:
run_multi_api_server(args)