[Bugfix] Fix FlashInfer all_reduce backend selection (#56891)

This commit is contained in:
Elvir Crnčević
2026-10-03 22:18:58 -04:00
committed by GitHub
parent 5e56e9af2a
commit ce49174247
2 changed files with 67 additions and 22 deletions
+28 -2
View File
@@ -376,7 +376,6 @@ def test_flashinfer_standalone_workspace_size(
"_resolve_fi_ar_backend", "_resolve_fi_ar_backend",
Mock(return_value=("mnnvl", False)), Mock(return_value=("mnnvl", False)),
) )
monkeypatch.setattr(flashinfer_all_reduce, "get_node_count", lambda: 2)
monkeypatch.setattr( monkeypatch.setattr(
flashinfer_all_reduce, flashinfer_all_reduce,
"_get_tuned_standalone_max_size", "_get_tuned_standalone_max_size",
@@ -389,6 +388,34 @@ def test_flashinfer_standalone_workspace_size(
assert create_workspace.call_args.args[3] == expected assert create_workspace.call_args.args[3] == expected
@pytest.mark.parametrize(
("node_count", "is_blackwell", "expected"),
[
(1, False, ("trtllm", False)),
(1, True, ("trtllm", False)),
(2, False, (None, False)),
(2, True, ("mnnvl", False)),
],
)
def test_flashinfer_auto_backend_uses_group_topology(
monkeypatch: pytest.MonkeyPatch,
node_count: int,
is_blackwell: bool,
expected: tuple[str | None, bool],
) -> None:
monkeypatch.setattr(
flashinfer_all_reduce.envs, "VLLM_FLASHINFER_ALLREDUCE_BACKEND", "auto"
)
monkeypatch.setattr(flashinfer_all_reduce, "_node_count", lambda _: node_count)
monkeypatch.setattr(
flashinfer_all_reduce.current_platform,
"has_device_capability",
lambda capability: is_blackwell,
)
assert flashinfer_all_reduce._resolve_fi_ar_backend(Mock()) == expected
def test_flashinfer_workspace_failure_is_not_retried( def test_flashinfer_workspace_failure_is_not_retried(
monkeypatch: pytest.MonkeyPatch, monkeypatch: pytest.MonkeyPatch,
) -> None: ) -> None:
@@ -402,7 +429,6 @@ def test_flashinfer_workspace_failure_is_not_retried(
"_resolve_fi_ar_backend", "_resolve_fi_ar_backend",
Mock(return_value=("mnnvl", True)), Mock(return_value=("mnnvl", True)),
) )
monkeypatch.setattr(flashinfer_all_reduce, "get_node_count", lambda: 1)
monkeypatch.setattr( monkeypatch.setattr(
flashinfer_all_reduce, "_get_tuned_standalone_max_size", Mock(return_value=None) flashinfer_all_reduce, "_get_tuned_standalone_max_size", Mock(return_value=None)
) )
@@ -17,7 +17,7 @@ from vllm.config.compilation import PassConfig
from vllm.distributed.device_communicators.all_reduce_utils import ( from vllm.distributed.device_communicators.all_reduce_utils import (
FI_MNNVL_ALLREDUCE_MAX_SIZE_MB, FI_MNNVL_ALLREDUCE_MAX_SIZE_MB,
) )
from vllm.distributed.parallel_state import _node_count, get_node_count from vllm.distributed.parallel_state import _node_count
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.platforms import current_platform from vllm.platforms import current_platform
from vllm.utils.gpu_sync_debug import gpu_sync_allowed from vllm.utils.gpu_sync_debug import gpu_sync_allowed
@@ -140,14 +140,13 @@ def _create_workspace(
return workspace return workspace
def _resolve_fi_ar_backend() -> tuple[str, bool]: def _resolve_fi_ar_backend(group: ProcessGroup) -> tuple[str | None, bool]:
"""Resolve the flashinfer allreduce backend for the current setup. """Resolve the flashinfer allreduce backend for the current setup.
Returns: Returns:
A ``(backend, allow_trtllm_fallback)`` tuple. ``allow_trtllm_fallback`` A ``(backend, allow_trtllm_fallback)`` tuple. ``backend`` is ``None``
is True only when ``auto`` selects mnnvl for a single node, so that when automatic selection has no supported backend for the process
workspace creation can fall back to trtllm on single-node topologies group.
without NVSwitch multicast support (where mnnvl is unavailable).
""" """
backend = envs.VLLM_FLASHINFER_ALLREDUCE_BACKEND backend = envs.VLLM_FLASHINFER_ALLREDUCE_BACKEND
@@ -155,15 +154,23 @@ def _resolve_fi_ar_backend() -> tuple[str, bool]:
logger.debug_once("Using flashinfer allreduce backend: %s", backend) logger.debug_once("Using flashinfer allreduce backend: %s", backend)
return backend, False return backend, False
# Default to mnnvl for both single- and multi-node setups. The mnnvl node_count = _node_count(group)
# cudagraph hang that previously forced single-node to trtllm if node_count == 1:
# (https://github.com/vllm-project/vllm/issues/35772) was fixed upstream in # TRTLLM supports single-node allreduce on Hopper and Blackwell. Do not
# FlashInfer (>= 0.6.12, vLLM pins 0.6.15), so mnnvl is safe here. trtllm # create an MNNVL workspace when the process group is node-local.
# does not support multi-node allreduce, so mnnvl is required there anyway. backend = "trtllm"
# mnnvl needs NVSwitch multicast; on single-node topologies without it, allow_trtllm_fallback = False
# fall back to trtllm so fused allreduce stays enabled. elif current_platform.has_device_capability(100):
backend = "mnnvl" # MNNVL requires Blackwell-class GPUs and is the only supported
allow_trtllm_fallback = get_node_count() == 1 # FlashInfer backend for a multi-node allreduce process group.
backend = "mnnvl"
allow_trtllm_fallback = False
else:
logger.debug_once(
"FlashInfer allreduce fusion is disabled: multi-node process groups "
"require an MNNVL-capable Blackwell system."
)
return None, False
logger.debug_once("Auto-selected flashinfer allreduce backend: %s", backend) logger.debug_once("Auto-selected flashinfer allreduce backend: %s", backend)
return backend, allow_trtllm_fallback return backend, allow_trtllm_fallback
@@ -187,9 +194,12 @@ def get_fi_ar_workspace(
if _fi_ar_workspace is not None or _fi_ar_workspace_failed: if _fi_ar_workspace is not None or _fi_ar_workspace_failed:
return _fi_ar_workspace return _fi_ar_workspace
backend, allow_trtllm_fallback = _resolve_fi_ar_backend() backend, allow_trtllm_fallback = _resolve_fi_ar_backend(group)
if get_node_count() > 1 and backend == "trtllm": if backend is None:
return None
if backend == "trtllm" and _node_count(group) > 1:
raise ValueError( raise ValueError(
"Flashinfer allreduce is not supported for multi-node allreduce with " "Flashinfer allreduce is not supported for multi-node allreduce with "
"'trtllm' backend. Please use 'mnnvl' backend instead." "'trtllm' backend. Please use 'mnnvl' backend instead."
@@ -253,9 +263,12 @@ def get_fi_ar_quant_workspace(
if _fi_ar_quant_workspace is not None or _fi_ar_quant_workspace_failed: if _fi_ar_quant_workspace is not None or _fi_ar_quant_workspace_failed:
return _fi_ar_quant_workspace return _fi_ar_quant_workspace
backend, allow_trtllm_fallback = _resolve_fi_ar_backend() backend, allow_trtllm_fallback = _resolve_fi_ar_backend(group)
if get_node_count() > 1 and backend == "trtllm": if backend is None:
return None
if backend == "trtllm" and _node_count(group) > 1:
raise ValueError( raise ValueError(
"Flashinfer allreduce quantization fusion is not supported for " "Flashinfer allreduce quantization fusion is not supported for "
"multi-node allreduce with 'trtllm' backend. Please use 'mnnvl' " "multi-node allreduce with 'trtllm' backend. Please use 'mnnvl' "
@@ -397,7 +410,13 @@ class FlashInferAllReduce:
self.world_size, self.world_size,
) )
return return
backend, _ = _resolve_fi_ar_backend() backend, _ = _resolve_fi_ar_backend(self.group)
if backend is None:
logger.info(
"FlashInfer All Reduce is disabled because no supported backend "
"is available for this process group."
)
return
tuned_max_size = _get_tuned_standalone_max_size( tuned_max_size = _get_tuned_standalone_max_size(
self.world_size, self.world_size,
backend, backend,