[CI] Fix B200 FusedMoE Layer Kernels hang on NVSHMEM IB setup (#61695)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
Misha Goin
2026-10-16 12:00:12 -04:00
committed by GitHub
co-authored by Claude Opus 5.5 Kevin H. Luu
parent 706791d516
commit cffc0cfa31
3 changed files with 28 additions and 4 deletions
+11 -1
View File
@@ -127,6 +127,14 @@ if has_nixl_ep():
BACKENDS += ["nixl_ep"]
DEEPEP_BACKENDS = {"deepep_high_throughput", "deepep_low_latency"}
# These tests run on one node, where NVSHMEM reaches peers over NVLink, so
# keep it off the NICs to avoid depending on IB setup on shared CI hosts.
DEEPEP_SINGLE_NODE_NVSHMEM_ENV = {
"NVSHMEM_REMOTE_TRANSPORT": "none",
# DeepEP forces IBGDA on, so also hide the NICs from it.
"NVSHMEM_HCA_LIST": "none",
}
MORI_BACKENDS = {"mori_high_throughput", "mori_low_latency"}
QUANT_METHODS = [
@@ -2187,6 +2195,8 @@ def test_moe_layer(
rocm_deepep = current_platform.is_rocm() and backend in DEEPEP_BACKENDS
parallel_worker = _parallel_worker_rocm_deepep if rocm_deepep else _parallel_worker
cuda_deepep = current_platform.is_cuda() and backend in DEEPEP_BACKENDS
env_dict = DEEPEP_SINGLE_NODE_NVSHMEM_ENV if cuda_deepep else None
launch_failures: list[str] = []
try:
@@ -2197,7 +2207,7 @@ def test_moe_layer(
world_size,
parallel_worker,
vllm_config,
None,
env_dict,
test_config_batch,
verbosity,
failure_report_path=failure_report_path,