[Bugfix][XPU] Skip oneCCL warm-up all_reduce when world_size == 1 (#52389)
Signed-off-by: DaeHyun Sung <dhsung@lablup.com>
This commit is contained in:
@@ -100,8 +100,13 @@ class XPUWorker(Worker):
|
|||||||
current_platform.dist_backend,
|
current_platform.dist_backend,
|
||||||
)
|
)
|
||||||
|
|
||||||
# global all_reduce needed for overall oneccl warm up
|
# oneCCL warm-up; only meaningful for multi-device runs. Requiring it
|
||||||
if torch.distributed.is_xccl_available():
|
# with a single worker breaks platforms where oneCCL cannot enumerate
|
||||||
|
# device topology (e.g. paravirtualized GPUs).
|
||||||
|
if (
|
||||||
|
self.parallel_config.world_size > 1
|
||||||
|
and torch.distributed.is_xccl_available()
|
||||||
|
):
|
||||||
torch.distributed.all_reduce(torch.zeros(1).xpu())
|
torch.distributed.all_reduce(torch.zeros(1).xpu())
|
||||||
|
|
||||||
if self.use_v2_model_runner:
|
if self.use_v2_model_runner:
|
||||||
|
|||||||
Reference in New Issue
Block a user