[BugFix] Dense multinode DP rescope with regression test (#49212)
Signed-off-by: Andy Lo <andy@mistral.ai> Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Andy Lo <andy@mistral.ai>
This commit is contained in:
@@ -382,6 +382,31 @@ def test_async_scheduling_with_pipeline_parallelism_is_allowed():
|
||||
assert cfg.scheduler_config.async_scheduling is True
|
||||
|
||||
|
||||
def test_reconfigure_for_independent_dp_rank_on_multinode_dense_model():
|
||||
parallel_config = ParallelConfig(
|
||||
tensor_parallel_size=8,
|
||||
data_parallel_size=2,
|
||||
data_parallel_size_local=1,
|
||||
data_parallel_rank=1,
|
||||
distributed_executor_backend="mp",
|
||||
nnodes=2,
|
||||
node_rank=1,
|
||||
)
|
||||
|
||||
assert parallel_config.nnodes_within_dp == 1
|
||||
assert parallel_config.node_rank_within_dp == 0
|
||||
|
||||
parallel_config.reconfigure_for_independent_dp_rank()
|
||||
|
||||
assert parallel_config.data_parallel_size == 1
|
||||
assert parallel_config.data_parallel_size_local == 1
|
||||
assert parallel_config.data_parallel_rank == 0
|
||||
assert parallel_config.data_parallel_index == 1
|
||||
assert parallel_config.nnodes == 1
|
||||
assert parallel_config.node_rank == 0
|
||||
assert parallel_config.world_size == 8
|
||||
|
||||
|
||||
@dataclass
|
||||
class _TestConfigFields:
|
||||
a: int
|
||||
|
||||
@@ -1034,3 +1034,14 @@ class ParallelConfig:
|
||||
)
|
||||
|
||||
return self
|
||||
|
||||
def reconfigure_for_independent_dp_rank(self) -> None:
|
||||
"""Reconfigure for a single independent non-MoE DP rank."""
|
||||
# Capture these before changing DP fields.
|
||||
nnodes = self.nnodes_within_dp
|
||||
node_rank = self.node_rank_within_dp
|
||||
self.data_parallel_size = 1
|
||||
self.data_parallel_size_local = 1
|
||||
self.data_parallel_rank = 0
|
||||
self.nnodes = nnodes
|
||||
self.node_rank = node_rank
|
||||
|
||||
@@ -1308,9 +1308,7 @@ class EngineCoreProc(EngineCore):
|
||||
# Non-MoE DP ranks are completely independent, so treat like DP=1.
|
||||
# Note that parallel_config.data_parallel_index will still reflect
|
||||
# the original DP rank.
|
||||
parallel_config.data_parallel_size = 1
|
||||
parallel_config.data_parallel_size_local = 1
|
||||
parallel_config.data_parallel_rank = 0
|
||||
parallel_config.reconfigure_for_independent_dp_rank()
|
||||
engine_core = EngineCoreProc(*args, engine_index=dp_rank, **kwargs)
|
||||
|
||||
assert engine_core is not None
|
||||
@@ -2466,10 +2464,7 @@ class EngineCoreActor(EngineCoreActorMixin, EngineCoreProc):
|
||||
dp_rank: int = 0,
|
||||
local_dp_rank: int = 0,
|
||||
):
|
||||
vllm_config.parallel_config.data_parallel_size = 1
|
||||
vllm_config.parallel_config.data_parallel_size_local = 1
|
||||
vllm_config.parallel_config.data_parallel_rank = 0
|
||||
|
||||
vllm_config.parallel_config.reconfigure_for_independent_dp_rank()
|
||||
EngineCoreActorMixin.__init__(
|
||||
self, vllm_config, addresses, dp_rank, local_dp_rank
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user