[CI] Fix B200 FusedMoE Layer Kernels hang on NVSHMEM IB setup (#61695)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
Misha Goin
2026-10-16 12:00:12 -04:00
committed by GitHub
co-authored by Claude Opus 5.5 Kevin H. Luu
parent 706791d516
commit cffc0cfa31
3 changed files with 28 additions and 4 deletions
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import dataclasses
import os
import time
import traceback
from collections.abc import Callable
from typing import Any, Concatenate
@@ -23,6 +24,8 @@ from vllm.utils.network_utils import get_open_port
P = ParamSpec("P")
LAUNCH_TIMEOUT_S = 900
@dataclasses.dataclass
class ProcessGroupInfo:
@@ -148,7 +151,7 @@ def parallel_launch_with_config(
*args: P.args,
**kwargs: P.kwargs,
) -> None:
spawn(
context = spawn(
_worker_parallel_launch,
args=(
world_size,
@@ -162,5 +165,16 @@ def parallel_launch_with_config(
)
+ args,
nprocs=world_size,
join=True,
join=False,
)
# A worker stuck in communicator setup never exits, so bound the wait.
deadline = time.monotonic() + LAUNCH_TIMEOUT_S
while not context.join(timeout=max(0.0, deadline - time.monotonic())):
if time.monotonic() >= deadline:
for process in context.processes:
process.kill()
for process in context.processes:
process.join()
raise TimeoutError(
f"Parallel workers did not finish within {LAUNCH_TIMEOUT_S}s"
)