diff --git a/docs/serving/data_parallel_deployment.md b/docs/serving/data_parallel_deployment.md index 7b963b99d5..1f18b92f95 100644 --- a/docs/serving/data_parallel_deployment.md +++ b/docs/serving/data_parallel_deployment.md @@ -98,7 +98,7 @@ For larger scale deployments especially, it can make sense to handle the orchest In this case, it's more convenient to treat each DP rank like a separate vLLM deployment, with its own endpoint, and have an external router balance HTTP requests between them, making use of appropriate real-time telemetry from each server for routing decisions. -This can already be done trivially for non-MoE models, since each deployed server is fully independent. No data parallel CLI options need to be used for this. +This can already be done trivially for non-MoE models, since each deployed server is fully independent. In that case, launch independent vLLM instances without any `--data-parallel-*` arguments; external DP CLI options are only supported for MoE deployments. We support an equivalent topology for MoE DP+EP which can be configured via the following CLI arguments. diff --git a/tests/test_config.py b/tests/test_config.py index 02e4d1d5d7..57d1e1bc68 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1215,8 +1215,6 @@ def test_scheduler_config_init(): ("facebook/opt-125m", 1, False, False), # Non-MoE model with DP>1 internal LB should need coordinator ("facebook/opt-125m", 2, False, True), - # Non-MoE model with DP>1 external LB should not need coordinator - ("facebook/opt-125m", 2, True, False), # MoE model with DP=1 should not need coordinator ("mistralai/Mixtral-8x7B-Instruct-v0.1", 1, False, False), # MoE model with DP>1 internal LB should need both coordinator diff --git a/tests/v1/distributed/test_external_lb_dp.py b/tests/v1/distributed/test_external_lb_dp.py index cfef8449eb..06e8e574a0 100644 --- a/tests/v1/distributed/test_external_lb_dp.py +++ b/tests/v1/distributed/test_external_lb_dp.py @@ -14,7 +14,7 @@ import requests from tests.utils import RemoteOpenAIServer from vllm.platforms import current_platform -MODEL_NAME = "ibm-research/PowerMoE-3b" +MODEL_NAME = os.getenv("MODEL_NAME", "ibm-research/PowerMoE-3b") # Number of data parallel ranks for external LB testing DP_SIZE = int(os.getenv("DP_SIZE", "2")) diff --git a/vllm/config/parallel.py b/vllm/config/parallel.py index 6ba392802e..95fd8787af 100644 --- a/vllm/config/parallel.py +++ b/vllm/config/parallel.py @@ -135,8 +135,10 @@ class ParallelConfig: data_parallel_external_lb: bool = False """Whether to use "external" DP LB mode. Applies only to online serving and when data_parallel_size > 0. This is useful for a "one-pod-per-rank" - wide-EP setup in Kubernetes. Set implicitly when --data-parallel-rank - is provided explicitly to vllm serve.""" + wide-EP setup in Kubernetes. Supported only for MoE deployments; non-MoE + models should use independent vLLM instances without --data-parallel-* + arguments. Set implicitly when --data-parallel-rank is provided explicitly + to vllm serve.""" data_parallel_hybrid_lb: bool = False """Whether to use "hybrid" DP LB mode. Applies only to online serving and when data_parallel_size > 0. Enables running an AsyncLLM diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index cd95510033..4c37d5a149 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -962,7 +962,9 @@ class EngineArgs: "-dpn", type=int, help="Data parallel rank of this instance. " - "When set, enables external load balancer mode.", + "When set, enables external load balancer mode for MoE " + "data-parallel deployments. Unsupported for non-MoE models; " + "launch independent vLLM instances instead.", ) parallel_group.add_argument( "--data-parallel-start-rank", @@ -1793,6 +1795,16 @@ class EngineArgs: data_parallel_external_lb = ( self.data_parallel_external_lb or self.data_parallel_rank is not None ) + if ( + self.data_parallel_size > 1 + and data_parallel_external_lb + and not model_config.is_moe + ): + raise ValueError( + "Non-MoE models do not support external data parallel mode. " + "For external load balancing, launch independent vLLM " + "instances without --data-parallel-* arguments." + ) # Local DP rank = 1, use pure-external LB. if data_parallel_external_lb: assert self.data_parallel_rank is not None, (