From ce49174247fd84c322f293a31cbe1d0460aaa7fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elvir=20Crn=C4=8Devi=C4=87?= Date: Sun, 4 Oct 2026 04:18:58 +0200 Subject: [PATCH] [Bugfix] Fix FlashInfer all_reduce backend selection (#56891) --- tests/distributed/test_comm_ops.py | 30 +++++++++- .../flashinfer_all_reduce.py | 59 ++++++++++++------- 2 files changed, 67 insertions(+), 22 deletions(-) diff --git a/tests/distributed/test_comm_ops.py b/tests/distributed/test_comm_ops.py index 25434a51b5..253ec8a188 100644 --- a/tests/distributed/test_comm_ops.py +++ b/tests/distributed/test_comm_ops.py @@ -376,7 +376,6 @@ def test_flashinfer_standalone_workspace_size( "_resolve_fi_ar_backend", Mock(return_value=("mnnvl", False)), ) - monkeypatch.setattr(flashinfer_all_reduce, "get_node_count", lambda: 2) monkeypatch.setattr( flashinfer_all_reduce, "_get_tuned_standalone_max_size", @@ -389,6 +388,34 @@ def test_flashinfer_standalone_workspace_size( assert create_workspace.call_args.args[3] == expected +@pytest.mark.parametrize( + ("node_count", "is_blackwell", "expected"), + [ + (1, False, ("trtllm", False)), + (1, True, ("trtllm", False)), + (2, False, (None, False)), + (2, True, ("mnnvl", False)), + ], +) +def test_flashinfer_auto_backend_uses_group_topology( + monkeypatch: pytest.MonkeyPatch, + node_count: int, + is_blackwell: bool, + expected: tuple[str | None, bool], +) -> None: + monkeypatch.setattr( + flashinfer_all_reduce.envs, "VLLM_FLASHINFER_ALLREDUCE_BACKEND", "auto" + ) + monkeypatch.setattr(flashinfer_all_reduce, "_node_count", lambda _: node_count) + monkeypatch.setattr( + flashinfer_all_reduce.current_platform, + "has_device_capability", + lambda capability: is_blackwell, + ) + + assert flashinfer_all_reduce._resolve_fi_ar_backend(Mock()) == expected + + def test_flashinfer_workspace_failure_is_not_retried( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -402,7 +429,6 @@ def test_flashinfer_workspace_failure_is_not_retried( "_resolve_fi_ar_backend", Mock(return_value=("mnnvl", True)), ) - monkeypatch.setattr(flashinfer_all_reduce, "get_node_count", lambda: 1) monkeypatch.setattr( flashinfer_all_reduce, "_get_tuned_standalone_max_size", Mock(return_value=None) ) diff --git a/vllm/distributed/device_communicators/flashinfer_all_reduce.py b/vllm/distributed/device_communicators/flashinfer_all_reduce.py index 9bb17e1528..d51e9e0c0e 100644 --- a/vllm/distributed/device_communicators/flashinfer_all_reduce.py +++ b/vllm/distributed/device_communicators/flashinfer_all_reduce.py @@ -17,7 +17,7 @@ from vllm.config.compilation import PassConfig from vllm.distributed.device_communicators.all_reduce_utils import ( FI_MNNVL_ALLREDUCE_MAX_SIZE_MB, ) -from vllm.distributed.parallel_state import _node_count, get_node_count +from vllm.distributed.parallel_state import _node_count from vllm.logger import init_logger from vllm.platforms import current_platform from vllm.utils.gpu_sync_debug import gpu_sync_allowed @@ -140,14 +140,13 @@ def _create_workspace( return workspace -def _resolve_fi_ar_backend() -> tuple[str, bool]: +def _resolve_fi_ar_backend(group: ProcessGroup) -> tuple[str | None, bool]: """Resolve the flashinfer allreduce backend for the current setup. Returns: - A ``(backend, allow_trtllm_fallback)`` tuple. ``allow_trtllm_fallback`` - is True only when ``auto`` selects mnnvl for a single node, so that - workspace creation can fall back to trtllm on single-node topologies - without NVSwitch multicast support (where mnnvl is unavailable). + A ``(backend, allow_trtllm_fallback)`` tuple. ``backend`` is ``None`` + when automatic selection has no supported backend for the process + group. """ backend = envs.VLLM_FLASHINFER_ALLREDUCE_BACKEND @@ -155,15 +154,23 @@ def _resolve_fi_ar_backend() -> tuple[str, bool]: logger.debug_once("Using flashinfer allreduce backend: %s", backend) return backend, False - # Default to mnnvl for both single- and multi-node setups. The mnnvl - # cudagraph hang that previously forced single-node to trtllm - # (https://github.com/vllm-project/vllm/issues/35772) was fixed upstream in - # FlashInfer (>= 0.6.12, vLLM pins 0.6.15), so mnnvl is safe here. trtllm - # does not support multi-node allreduce, so mnnvl is required there anyway. - # mnnvl needs NVSwitch multicast; on single-node topologies without it, - # fall back to trtllm so fused allreduce stays enabled. - backend = "mnnvl" - allow_trtllm_fallback = get_node_count() == 1 + node_count = _node_count(group) + if node_count == 1: + # TRTLLM supports single-node allreduce on Hopper and Blackwell. Do not + # create an MNNVL workspace when the process group is node-local. + backend = "trtllm" + allow_trtllm_fallback = False + elif current_platform.has_device_capability(100): + # MNNVL requires Blackwell-class GPUs and is the only supported + # FlashInfer backend for a multi-node allreduce process group. + backend = "mnnvl" + allow_trtllm_fallback = False + else: + logger.debug_once( + "FlashInfer allreduce fusion is disabled: multi-node process groups " + "require an MNNVL-capable Blackwell system." + ) + return None, False logger.debug_once("Auto-selected flashinfer allreduce backend: %s", backend) return backend, allow_trtllm_fallback @@ -187,9 +194,12 @@ def get_fi_ar_workspace( if _fi_ar_workspace is not None or _fi_ar_workspace_failed: return _fi_ar_workspace - backend, allow_trtllm_fallback = _resolve_fi_ar_backend() + backend, allow_trtllm_fallback = _resolve_fi_ar_backend(group) - if get_node_count() > 1 and backend == "trtllm": + if backend is None: + return None + + if backend == "trtllm" and _node_count(group) > 1: raise ValueError( "Flashinfer allreduce is not supported for multi-node allreduce with " "'trtllm' backend. Please use 'mnnvl' backend instead." @@ -253,9 +263,12 @@ def get_fi_ar_quant_workspace( if _fi_ar_quant_workspace is not None or _fi_ar_quant_workspace_failed: return _fi_ar_quant_workspace - backend, allow_trtllm_fallback = _resolve_fi_ar_backend() + backend, allow_trtllm_fallback = _resolve_fi_ar_backend(group) - if get_node_count() > 1 and backend == "trtllm": + if backend is None: + return None + + if backend == "trtllm" and _node_count(group) > 1: raise ValueError( "Flashinfer allreduce quantization fusion is not supported for " "multi-node allreduce with 'trtllm' backend. Please use 'mnnvl' " @@ -397,7 +410,13 @@ class FlashInferAllReduce: self.world_size, ) return - backend, _ = _resolve_fi_ar_backend() + backend, _ = _resolve_fi_ar_backend(self.group) + if backend is None: + logger.info( + "FlashInfer All Reduce is disabled because no supported backend " + "is available for this process group." + ) + return tuned_max_size = _get_tuned_standalone_max_size( self.world_size, backend,