[Distributed] Default FlashInfer allreduce to mnnvl on single node (#47219)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Woosuk Kwon
2026-06-30 18:35:56 -07:00
committed by GitHub
co-authored by Claude Opus 4.8
parent f098ee70c7
commit a264e41975
@@ -92,25 +92,32 @@ def _create_workspace(
return workspace
def _resolve_fi_ar_backend() -> str:
def _resolve_fi_ar_backend() -> tuple[str, bool]:
"""Resolve the flashinfer allreduce backend for the current setup.
Returns:
A ``(backend, allow_trtllm_fallback)`` tuple. ``allow_trtllm_fallback``
is True only when ``auto`` selects mnnvl for a single node, so that
workspace creation can fall back to trtllm on single-node topologies
without NVSwitch multicast support (where mnnvl is unavailable).
"""
backend = envs.VLLM_FLASHINFER_ALLREDUCE_BACKEND
if backend != "auto":
logger.info_once(f"Using flashinfer allreduce backend: {backend}")
return backend
return backend, False
if get_node_count() > 1: # noqa: SIM108
# Use mnnvl backend for multi-node setup since
# trtllm backend does not support multi-node allreduce
backend = "mnnvl"
else:
# Currently defaulting to trtllm backend for single-node
# setup since mnnvl has issues with cudagraph:
# https://github.com/vllm-project/vllm/issues/35772
# Should switch back to auto when the issue is resolved.
backend = "trtllm"
# Default to mnnvl for both single- and multi-node setups. The mnnvl
# cudagraph hang that previously forced single-node to trtllm
# (https://github.com/vllm-project/vllm/issues/35772) was fixed upstream in
# FlashInfer (>= 0.6.12, vLLM pins 0.6.13), so mnnvl is safe here. trtllm
# does not support multi-node allreduce, so mnnvl is required there anyway.
# mnnvl needs NVSwitch multicast; on single-node topologies without it,
# fall back to trtllm so fused allreduce stays enabled.
backend = "mnnvl"
allow_trtllm_fallback = get_node_count() == 1
logger.info_once(f"Auto-selected flashinfer allreduce backend: {backend}")
return backend
return backend, allow_trtllm_fallback
def get_fi_ar_workspace(
@@ -132,7 +139,7 @@ def get_fi_ar_workspace(
if _fi_ar_workspace is not None:
return _fi_ar_workspace
backend = _resolve_fi_ar_backend()
backend, allow_trtllm_fallback = _resolve_fi_ar_backend()
if get_node_count() > 1 and backend == "trtllm":
raise ValueError(
@@ -140,14 +147,23 @@ def get_fi_ar_workspace(
"'trtllm' backend. Please use 'mnnvl' backend instead."
)
# Reuse the quant workspace if it was already created with the same backend
if _fi_ar_quant_workspace is not None and _fi_ar_quant_workspace.backend == backend:
_fi_ar_workspace = _fi_ar_quant_workspace
return _fi_ar_workspace
def _get_or_create(be: str):
# Reuse the quant workspace if it was already created with the same backend
if _fi_ar_quant_workspace is not None and _fi_ar_quant_workspace.backend == be:
return _fi_ar_quant_workspace
return _create_workspace(
be, world_size, rank, max_token_num, hidden_dim, dtype, group
)
_fi_ar_workspace = _get_or_create(backend)
if _fi_ar_workspace is None and allow_trtllm_fallback and backend != "trtllm":
logger.warning_once(
"FlashInfer mnnvl allreduce workspace unavailable (likely no NVSwitch "
"multicast support); falling back to trtllm backend for single node."
)
backend = "trtllm"
_fi_ar_workspace = _get_or_create(backend)
_fi_ar_workspace = _create_workspace(
backend, world_size, rank, max_token_num, hidden_dim, dtype, group
)
if _fi_ar_workspace is not None:
logger.info_once(
"Initialized FlashInfer Allreduce norm fusion workspace "