[XPU] Fix device assignment for DP external LB (#53037)
Signed-off-by: Tony Lin <tony.lin@intel.com>
This commit is contained in:
@@ -47,18 +47,74 @@ class XPUWorker(Worker):
|
||||
parallel_config.distributed_executor_backend
|
||||
not in ("ray", "external_launcher")
|
||||
and parallel_config.data_parallel_backend != "ray"
|
||||
and parallel_config.nnodes_within_dp == 1
|
||||
and (
|
||||
parallel_config.data_parallel_external_lb
|
||||
or parallel_config.nnodes_within_dp == 1
|
||||
)
|
||||
):
|
||||
dp_local_rank = parallel_config.data_parallel_rank_local
|
||||
if dp_local_rank is None:
|
||||
dp_local_rank = parallel_config.data_parallel_index
|
||||
tp_pp_world_size = (
|
||||
parallel_config.pipeline_parallel_size
|
||||
* parallel_config.tensor_parallel_size
|
||||
)
|
||||
self.local_rank += dp_local_rank * tp_pp_world_size
|
||||
|
||||
replica_world_size = parallel_config.world_size
|
||||
visible_device_count = torch.accelerator.device_count()
|
||||
|
||||
if parallel_config.data_parallel_external_lb:
|
||||
if parallel_config.nnodes_within_dp > 1:
|
||||
# The replica spans nodes, so use its node-local shard.
|
||||
replica_world_size = parallel_config.local_world_size
|
||||
if replica_world_size > visible_device_count:
|
||||
raise ValueError(
|
||||
f"Local TP/PP/PCP replica size ({replica_world_size}) "
|
||||
"exceeds the number of visible XPU devices "
|
||||
f"({visible_device_count})."
|
||||
)
|
||||
local_dp_capacity = visible_device_count // replica_world_size
|
||||
elif replica_world_size < visible_device_count:
|
||||
# A node can host multiple complete TP/PP/PCP replicas.
|
||||
local_dp_capacity = visible_device_count // replica_world_size
|
||||
if visible_device_count % replica_world_size != 0:
|
||||
logger.warning_once(
|
||||
"XPU external LB cannot evenly divide "
|
||||
"%d visible devices into TP/PP/PCP replicas of "
|
||||
"size %d. This node can host %d complete DP "
|
||||
"replicas, leaving %d visible devices unused.",
|
||||
visible_device_count,
|
||||
replica_world_size,
|
||||
local_dp_capacity,
|
||||
visible_device_count % replica_world_size,
|
||||
)
|
||||
elif replica_world_size == visible_device_count:
|
||||
# A node hosts exactly one complete TP/PP/PCP replica.
|
||||
local_dp_capacity = 1
|
||||
logger.warning_once(
|
||||
"XPU external LB sees exactly enough devices for one "
|
||||
"TP/PP/PCP replica. This may be the intended "
|
||||
"configuration, but it may also indicate that device "
|
||||
"visibility is misconfigured. Every DP rank must see "
|
||||
"all XPU devices on its node; consider removing "
|
||||
"ZE_AFFINITY_MASK or setting it to expose the complete "
|
||||
"device set."
|
||||
)
|
||||
else:
|
||||
# The topology says single-node, but the replica does not fit.
|
||||
raise ValueError(
|
||||
f"TP/PP/PCP replica size ({replica_world_size}) exceeds "
|
||||
f"the number of visible XPU devices ({visible_device_count}), "
|
||||
"but nnodes_within_dp is 1. Configure the multi-node "
|
||||
"topology, or ensure every DP rank can see all devices "
|
||||
"on its node."
|
||||
)
|
||||
# Strip the node component off the global DP index to get this
|
||||
# engine's slot on its own node. Assumes the launcher assigns
|
||||
# DP ranks to nodes in contiguous blocks (node 0 gets ranks
|
||||
# 0..capacity-1, and so on), which is what the usual sequential
|
||||
# and one-pod-per-rank deployments do. A round-robin or
|
||||
# unbalanced assignment would silently map two engines onto the
|
||||
# same device.
|
||||
dp_local_rank = parallel_config.data_parallel_index % local_dp_capacity
|
||||
|
||||
self.local_rank += dp_local_rank * replica_world_size
|
||||
|
||||
assert self.local_rank < visible_device_count, (
|
||||
f"DP adjusted local rank {self.local_rank} is out of bounds. "
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user