[Distributed] Default FlashInfer allreduce to mnnvl on single node (#47219)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
f098ee70c7
commit
a264e41975
@@ -92,25 +92,32 @@ def _create_workspace(
|
||||
return workspace
|
||||
|
||||
|
||||
def _resolve_fi_ar_backend() -> str:
|
||||
def _resolve_fi_ar_backend() -> tuple[str, bool]:
|
||||
"""Resolve the flashinfer allreduce backend for the current setup.
|
||||
|
||||
Returns:
|
||||
A ``(backend, allow_trtllm_fallback)`` tuple. ``allow_trtllm_fallback``
|
||||
is True only when ``auto`` selects mnnvl for a single node, so that
|
||||
workspace creation can fall back to trtllm on single-node topologies
|
||||
without NVSwitch multicast support (where mnnvl is unavailable).
|
||||
"""
|
||||
backend = envs.VLLM_FLASHINFER_ALLREDUCE_BACKEND
|
||||
if backend != "auto":
|
||||
logger.info_once(f"Using flashinfer allreduce backend: {backend}")
|
||||
return backend
|
||||
return backend, False
|
||||
|
||||
if get_node_count() > 1: # noqa: SIM108
|
||||
# Use mnnvl backend for multi-node setup since
|
||||
# trtllm backend does not support multi-node allreduce
|
||||
backend = "mnnvl"
|
||||
else:
|
||||
# Currently defaulting to trtllm backend for single-node
|
||||
# setup since mnnvl has issues with cudagraph:
|
||||
# https://github.com/vllm-project/vllm/issues/35772
|
||||
# Should switch back to auto when the issue is resolved.
|
||||
backend = "trtllm"
|
||||
# Default to mnnvl for both single- and multi-node setups. The mnnvl
|
||||
# cudagraph hang that previously forced single-node to trtllm
|
||||
# (https://github.com/vllm-project/vllm/issues/35772) was fixed upstream in
|
||||
# FlashInfer (>= 0.6.12, vLLM pins 0.6.13), so mnnvl is safe here. trtllm
|
||||
# does not support multi-node allreduce, so mnnvl is required there anyway.
|
||||
# mnnvl needs NVSwitch multicast; on single-node topologies without it,
|
||||
# fall back to trtllm so fused allreduce stays enabled.
|
||||
backend = "mnnvl"
|
||||
allow_trtllm_fallback = get_node_count() == 1
|
||||
|
||||
logger.info_once(f"Auto-selected flashinfer allreduce backend: {backend}")
|
||||
return backend
|
||||
return backend, allow_trtllm_fallback
|
||||
|
||||
|
||||
def get_fi_ar_workspace(
|
||||
@@ -132,7 +139,7 @@ def get_fi_ar_workspace(
|
||||
if _fi_ar_workspace is not None:
|
||||
return _fi_ar_workspace
|
||||
|
||||
backend = _resolve_fi_ar_backend()
|
||||
backend, allow_trtllm_fallback = _resolve_fi_ar_backend()
|
||||
|
||||
if get_node_count() > 1 and backend == "trtllm":
|
||||
raise ValueError(
|
||||
@@ -140,14 +147,23 @@ def get_fi_ar_workspace(
|
||||
"'trtllm' backend. Please use 'mnnvl' backend instead."
|
||||
)
|
||||
|
||||
# Reuse the quant workspace if it was already created with the same backend
|
||||
if _fi_ar_quant_workspace is not None and _fi_ar_quant_workspace.backend == backend:
|
||||
_fi_ar_workspace = _fi_ar_quant_workspace
|
||||
return _fi_ar_workspace
|
||||
def _get_or_create(be: str):
|
||||
# Reuse the quant workspace if it was already created with the same backend
|
||||
if _fi_ar_quant_workspace is not None and _fi_ar_quant_workspace.backend == be:
|
||||
return _fi_ar_quant_workspace
|
||||
return _create_workspace(
|
||||
be, world_size, rank, max_token_num, hidden_dim, dtype, group
|
||||
)
|
||||
|
||||
_fi_ar_workspace = _get_or_create(backend)
|
||||
if _fi_ar_workspace is None and allow_trtllm_fallback and backend != "trtllm":
|
||||
logger.warning_once(
|
||||
"FlashInfer mnnvl allreduce workspace unavailable (likely no NVSwitch "
|
||||
"multicast support); falling back to trtllm backend for single node."
|
||||
)
|
||||
backend = "trtllm"
|
||||
_fi_ar_workspace = _get_or_create(backend)
|
||||
|
||||
_fi_ar_workspace = _create_workspace(
|
||||
backend, world_size, rank, max_token_num, hidden_dim, dtype, group
|
||||
)
|
||||
if _fi_ar_workspace is not None:
|
||||
logger.info_once(
|
||||
"Initialized FlashInfer Allreduce norm fusion workspace "
|
||||
|
||||
Reference in New Issue
Block a user