[ROCm][Bugfix] Fix elastic EP scaling deadlock (#56610)

Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Matt
2026-09-12 23:09:19 +00:00
committed by GitHub
co-authored by Claude Opus 4.8
parent 7ee8a6dd01
commit 72d4d83157
4 changed files with 106 additions and 38 deletions
@@ -3,6 +3,7 @@
# ===================== import region =====================
import contextlib
import threading
import torch
@@ -26,6 +27,33 @@ from vllm.utils.torch_utils import current_stream
logger = init_logger(__name__)
_warmup_tls = threading.local()
def _warmup_deferred() -> bool:
return getattr(_warmup_tls, "deferred", False)
@contextlib.contextmanager
def defer_comm_warmup_on_rocm():
"""Build communicators on this thread without their warm-up on ROCm, where
the warm-up can deadlock if built next to a live engine whose existing
communicators still have collectives in flight; instead the warm-up is
deferred to when the engine is quiet (during commit).
"""
from vllm.platforms import current_platform
if not current_platform.is_rocm():
yield
return
prev = _warmup_deferred()
_warmup_tls.deferred = True
try:
yield
finally:
_warmup_tls.deferred = prev
_NCCL_SYMM_OPS_REGISTERED = False
_NCCL_SUSPEND_MEM = 0x01
@@ -153,6 +181,9 @@ class PyNcclCommunicator:
self.world_size, self.unique_id, self.rank
)
if _warmup_deferred():
return
stream = current_stream()
# A small all_reduce for warmup.
data = torch.zeros(1, device=device)