diff --git a/vllm/distributed/device_communicators/flashinfer_all_reduce.py b/vllm/distributed/device_communicators/flashinfer_all_reduce.py index 38f7bd5ff8..86a8524b68 100644 --- a/vllm/distributed/device_communicators/flashinfer_all_reduce.py +++ b/vllm/distributed/device_communicators/flashinfer_all_reduce.py @@ -92,25 +92,32 @@ def _create_workspace( return workspace -def _resolve_fi_ar_backend() -> str: +def _resolve_fi_ar_backend() -> tuple[str, bool]: + """Resolve the flashinfer allreduce backend for the current setup. + + Returns: + A ``(backend, allow_trtllm_fallback)`` tuple. ``allow_trtllm_fallback`` + is True only when ``auto`` selects mnnvl for a single node, so that + workspace creation can fall back to trtllm on single-node topologies + without NVSwitch multicast support (where mnnvl is unavailable). + """ backend = envs.VLLM_FLASHINFER_ALLREDUCE_BACKEND if backend != "auto": logger.info_once(f"Using flashinfer allreduce backend: {backend}") - return backend + return backend, False - if get_node_count() > 1: # noqa: SIM108 - # Use mnnvl backend for multi-node setup since - # trtllm backend does not support multi-node allreduce - backend = "mnnvl" - else: - # Currently defaulting to trtllm backend for single-node - # setup since mnnvl has issues with cudagraph: - # https://github.com/vllm-project/vllm/issues/35772 - # Should switch back to auto when the issue is resolved. - backend = "trtllm" + # Default to mnnvl for both single- and multi-node setups. The mnnvl + # cudagraph hang that previously forced single-node to trtllm + # (https://github.com/vllm-project/vllm/issues/35772) was fixed upstream in + # FlashInfer (>= 0.6.12, vLLM pins 0.6.13), so mnnvl is safe here. trtllm + # does not support multi-node allreduce, so mnnvl is required there anyway. + # mnnvl needs NVSwitch multicast; on single-node topologies without it, + # fall back to trtllm so fused allreduce stays enabled. + backend = "mnnvl" + allow_trtllm_fallback = get_node_count() == 1 logger.info_once(f"Auto-selected flashinfer allreduce backend: {backend}") - return backend + return backend, allow_trtllm_fallback def get_fi_ar_workspace( @@ -132,7 +139,7 @@ def get_fi_ar_workspace( if _fi_ar_workspace is not None: return _fi_ar_workspace - backend = _resolve_fi_ar_backend() + backend, allow_trtllm_fallback = _resolve_fi_ar_backend() if get_node_count() > 1 and backend == "trtllm": raise ValueError( @@ -140,14 +147,23 @@ def get_fi_ar_workspace( "'trtllm' backend. Please use 'mnnvl' backend instead." ) - # Reuse the quant workspace if it was already created with the same backend - if _fi_ar_quant_workspace is not None and _fi_ar_quant_workspace.backend == backend: - _fi_ar_workspace = _fi_ar_quant_workspace - return _fi_ar_workspace + def _get_or_create(be: str): + # Reuse the quant workspace if it was already created with the same backend + if _fi_ar_quant_workspace is not None and _fi_ar_quant_workspace.backend == be: + return _fi_ar_quant_workspace + return _create_workspace( + be, world_size, rank, max_token_num, hidden_dim, dtype, group + ) + + _fi_ar_workspace = _get_or_create(backend) + if _fi_ar_workspace is None and allow_trtllm_fallback and backend != "trtllm": + logger.warning_once( + "FlashInfer mnnvl allreduce workspace unavailable (likely no NVSwitch " + "multicast support); falling back to trtllm backend for single node." + ) + backend = "trtllm" + _fi_ar_workspace = _get_or_create(backend) - _fi_ar_workspace = _create_workspace( - backend, world_size, rank, max_token_num, hidden_dim, dtype, group - ) if _fi_ar_workspace is not None: logger.info_once( "Initialized FlashInfer Allreduce norm fusion workspace "