[ROCm][Bugfix] Fix elastic EP scaling deadlock (#56610)
Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -3,6 +3,7 @@
|
||||
|
||||
|
||||
# ===================== import region =====================
|
||||
import contextlib
|
||||
import threading
|
||||
|
||||
import torch
|
||||
@@ -26,6 +27,33 @@ from vllm.utils.torch_utils import current_stream
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_warmup_tls = threading.local()
|
||||
|
||||
|
||||
def _warmup_deferred() -> bool:
|
||||
return getattr(_warmup_tls, "deferred", False)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def defer_comm_warmup_on_rocm():
|
||||
"""Build communicators on this thread without their warm-up on ROCm, where
|
||||
the warm-up can deadlock if built next to a live engine whose existing
|
||||
communicators still have collectives in flight; instead the warm-up is
|
||||
deferred to when the engine is quiet (during commit).
|
||||
"""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
yield
|
||||
return
|
||||
prev = _warmup_deferred()
|
||||
_warmup_tls.deferred = True
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_warmup_tls.deferred = prev
|
||||
|
||||
|
||||
_NCCL_SYMM_OPS_REGISTERED = False
|
||||
|
||||
_NCCL_SUSPEND_MEM = 0x01
|
||||
@@ -153,6 +181,9 @@ class PyNcclCommunicator:
|
||||
self.world_size, self.unique_id, self.rank
|
||||
)
|
||||
|
||||
if _warmup_deferred():
|
||||
return
|
||||
|
||||
stream = current_stream()
|
||||
# A small all_reduce for warmup.
|
||||
data = torch.zeros(1, device=device)
|
||||
|
||||
Reference in New Issue
Block a user