[CI] Fix B200 FusedMoE Layer Kernels hang on NVSHMEM IB setup (#61695)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Kevin H. Luu
parent
706791d516
commit
cffc0cfa31
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import dataclasses
|
||||
import os
|
||||
import time
|
||||
import traceback
|
||||
from collections.abc import Callable
|
||||
from typing import Any, Concatenate
|
||||
@@ -23,6 +24,8 @@ from vllm.utils.network_utils import get_open_port
|
||||
|
||||
P = ParamSpec("P")
|
||||
|
||||
LAUNCH_TIMEOUT_S = 900
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class ProcessGroupInfo:
|
||||
@@ -148,7 +151,7 @@ def parallel_launch_with_config(
|
||||
*args: P.args,
|
||||
**kwargs: P.kwargs,
|
||||
) -> None:
|
||||
spawn(
|
||||
context = spawn(
|
||||
_worker_parallel_launch,
|
||||
args=(
|
||||
world_size,
|
||||
@@ -162,5 +165,16 @@ def parallel_launch_with_config(
|
||||
)
|
||||
+ args,
|
||||
nprocs=world_size,
|
||||
join=True,
|
||||
join=False,
|
||||
)
|
||||
# A worker stuck in communicator setup never exits, so bound the wait.
|
||||
deadline = time.monotonic() + LAUNCH_TIMEOUT_S
|
||||
while not context.join(timeout=max(0.0, deadline - time.monotonic())):
|
||||
if time.monotonic() >= deadline:
|
||||
for process in context.processes:
|
||||
process.kill()
|
||||
for process in context.processes:
|
||||
process.join()
|
||||
raise TimeoutError(
|
||||
f"Parallel workers did not finish within {LAUNCH_TIMEOUT_S}s"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user