[Bugfix] Fix FlashInfer all_reduce backend selection (#56891)
This commit is contained in:
@@ -376,7 +376,6 @@ def test_flashinfer_standalone_workspace_size(
|
|||||||
"_resolve_fi_ar_backend",
|
"_resolve_fi_ar_backend",
|
||||||
Mock(return_value=("mnnvl", False)),
|
Mock(return_value=("mnnvl", False)),
|
||||||
)
|
)
|
||||||
monkeypatch.setattr(flashinfer_all_reduce, "get_node_count", lambda: 2)
|
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
flashinfer_all_reduce,
|
flashinfer_all_reduce,
|
||||||
"_get_tuned_standalone_max_size",
|
"_get_tuned_standalone_max_size",
|
||||||
@@ -389,6 +388,34 @@ def test_flashinfer_standalone_workspace_size(
|
|||||||
assert create_workspace.call_args.args[3] == expected
|
assert create_workspace.call_args.args[3] == expected
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("node_count", "is_blackwell", "expected"),
|
||||||
|
[
|
||||||
|
(1, False, ("trtllm", False)),
|
||||||
|
(1, True, ("trtllm", False)),
|
||||||
|
(2, False, (None, False)),
|
||||||
|
(2, True, ("mnnvl", False)),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_flashinfer_auto_backend_uses_group_topology(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
node_count: int,
|
||||||
|
is_blackwell: bool,
|
||||||
|
expected: tuple[str | None, bool],
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setattr(
|
||||||
|
flashinfer_all_reduce.envs, "VLLM_FLASHINFER_ALLREDUCE_BACKEND", "auto"
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(flashinfer_all_reduce, "_node_count", lambda _: node_count)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
flashinfer_all_reduce.current_platform,
|
||||||
|
"has_device_capability",
|
||||||
|
lambda capability: is_blackwell,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert flashinfer_all_reduce._resolve_fi_ar_backend(Mock()) == expected
|
||||||
|
|
||||||
|
|
||||||
def test_flashinfer_workspace_failure_is_not_retried(
|
def test_flashinfer_workspace_failure_is_not_retried(
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
) -> None:
|
) -> None:
|
||||||
@@ -402,7 +429,6 @@ def test_flashinfer_workspace_failure_is_not_retried(
|
|||||||
"_resolve_fi_ar_backend",
|
"_resolve_fi_ar_backend",
|
||||||
Mock(return_value=("mnnvl", True)),
|
Mock(return_value=("mnnvl", True)),
|
||||||
)
|
)
|
||||||
monkeypatch.setattr(flashinfer_all_reduce, "get_node_count", lambda: 1)
|
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
flashinfer_all_reduce, "_get_tuned_standalone_max_size", Mock(return_value=None)
|
flashinfer_all_reduce, "_get_tuned_standalone_max_size", Mock(return_value=None)
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ from vllm.config.compilation import PassConfig
|
|||||||
from vllm.distributed.device_communicators.all_reduce_utils import (
|
from vllm.distributed.device_communicators.all_reduce_utils import (
|
||||||
FI_MNNVL_ALLREDUCE_MAX_SIZE_MB,
|
FI_MNNVL_ALLREDUCE_MAX_SIZE_MB,
|
||||||
)
|
)
|
||||||
from vllm.distributed.parallel_state import _node_count, get_node_count
|
from vllm.distributed.parallel_state import _node_count
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||||
@@ -140,14 +140,13 @@ def _create_workspace(
|
|||||||
return workspace
|
return workspace
|
||||||
|
|
||||||
|
|
||||||
def _resolve_fi_ar_backend() -> tuple[str, bool]:
|
def _resolve_fi_ar_backend(group: ProcessGroup) -> tuple[str | None, bool]:
|
||||||
"""Resolve the flashinfer allreduce backend for the current setup.
|
"""Resolve the flashinfer allreduce backend for the current setup.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
A ``(backend, allow_trtllm_fallback)`` tuple. ``allow_trtllm_fallback``
|
A ``(backend, allow_trtllm_fallback)`` tuple. ``backend`` is ``None``
|
||||||
is True only when ``auto`` selects mnnvl for a single node, so that
|
when automatic selection has no supported backend for the process
|
||||||
workspace creation can fall back to trtllm on single-node topologies
|
group.
|
||||||
without NVSwitch multicast support (where mnnvl is unavailable).
|
|
||||||
|
|
||||||
"""
|
"""
|
||||||
backend = envs.VLLM_FLASHINFER_ALLREDUCE_BACKEND
|
backend = envs.VLLM_FLASHINFER_ALLREDUCE_BACKEND
|
||||||
@@ -155,15 +154,23 @@ def _resolve_fi_ar_backend() -> tuple[str, bool]:
|
|||||||
logger.debug_once("Using flashinfer allreduce backend: %s", backend)
|
logger.debug_once("Using flashinfer allreduce backend: %s", backend)
|
||||||
return backend, False
|
return backend, False
|
||||||
|
|
||||||
# Default to mnnvl for both single- and multi-node setups. The mnnvl
|
node_count = _node_count(group)
|
||||||
# cudagraph hang that previously forced single-node to trtllm
|
if node_count == 1:
|
||||||
# (https://github.com/vllm-project/vllm/issues/35772) was fixed upstream in
|
# TRTLLM supports single-node allreduce on Hopper and Blackwell. Do not
|
||||||
# FlashInfer (>= 0.6.12, vLLM pins 0.6.15), so mnnvl is safe here. trtllm
|
# create an MNNVL workspace when the process group is node-local.
|
||||||
# does not support multi-node allreduce, so mnnvl is required there anyway.
|
backend = "trtllm"
|
||||||
# mnnvl needs NVSwitch multicast; on single-node topologies without it,
|
allow_trtllm_fallback = False
|
||||||
# fall back to trtllm so fused allreduce stays enabled.
|
elif current_platform.has_device_capability(100):
|
||||||
backend = "mnnvl"
|
# MNNVL requires Blackwell-class GPUs and is the only supported
|
||||||
allow_trtllm_fallback = get_node_count() == 1
|
# FlashInfer backend for a multi-node allreduce process group.
|
||||||
|
backend = "mnnvl"
|
||||||
|
allow_trtllm_fallback = False
|
||||||
|
else:
|
||||||
|
logger.debug_once(
|
||||||
|
"FlashInfer allreduce fusion is disabled: multi-node process groups "
|
||||||
|
"require an MNNVL-capable Blackwell system."
|
||||||
|
)
|
||||||
|
return None, False
|
||||||
|
|
||||||
logger.debug_once("Auto-selected flashinfer allreduce backend: %s", backend)
|
logger.debug_once("Auto-selected flashinfer allreduce backend: %s", backend)
|
||||||
return backend, allow_trtllm_fallback
|
return backend, allow_trtllm_fallback
|
||||||
@@ -187,9 +194,12 @@ def get_fi_ar_workspace(
|
|||||||
if _fi_ar_workspace is not None or _fi_ar_workspace_failed:
|
if _fi_ar_workspace is not None or _fi_ar_workspace_failed:
|
||||||
return _fi_ar_workspace
|
return _fi_ar_workspace
|
||||||
|
|
||||||
backend, allow_trtllm_fallback = _resolve_fi_ar_backend()
|
backend, allow_trtllm_fallback = _resolve_fi_ar_backend(group)
|
||||||
|
|
||||||
if get_node_count() > 1 and backend == "trtllm":
|
if backend is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
if backend == "trtllm" and _node_count(group) > 1:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
"Flashinfer allreduce is not supported for multi-node allreduce with "
|
"Flashinfer allreduce is not supported for multi-node allreduce with "
|
||||||
"'trtllm' backend. Please use 'mnnvl' backend instead."
|
"'trtllm' backend. Please use 'mnnvl' backend instead."
|
||||||
@@ -253,9 +263,12 @@ def get_fi_ar_quant_workspace(
|
|||||||
if _fi_ar_quant_workspace is not None or _fi_ar_quant_workspace_failed:
|
if _fi_ar_quant_workspace is not None or _fi_ar_quant_workspace_failed:
|
||||||
return _fi_ar_quant_workspace
|
return _fi_ar_quant_workspace
|
||||||
|
|
||||||
backend, allow_trtllm_fallback = _resolve_fi_ar_backend()
|
backend, allow_trtllm_fallback = _resolve_fi_ar_backend(group)
|
||||||
|
|
||||||
if get_node_count() > 1 and backend == "trtllm":
|
if backend is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
if backend == "trtllm" and _node_count(group) > 1:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
"Flashinfer allreduce quantization fusion is not supported for "
|
"Flashinfer allreduce quantization fusion is not supported for "
|
||||||
"multi-node allreduce with 'trtllm' backend. Please use 'mnnvl' "
|
"multi-node allreduce with 'trtllm' backend. Please use 'mnnvl' "
|
||||||
@@ -397,7 +410,13 @@ class FlashInferAllReduce:
|
|||||||
self.world_size,
|
self.world_size,
|
||||||
)
|
)
|
||||||
return
|
return
|
||||||
backend, _ = _resolve_fi_ar_backend()
|
backend, _ = _resolve_fi_ar_backend(self.group)
|
||||||
|
if backend is None:
|
||||||
|
logger.info(
|
||||||
|
"FlashInfer All Reduce is disabled because no supported backend "
|
||||||
|
"is available for this process group."
|
||||||
|
)
|
||||||
|
return
|
||||||
tuned_max_size = _get_tuned_standalone_max_size(
|
tuned_max_size = _get_tuned_standalone_max_size(
|
||||||
self.world_size,
|
self.world_size,
|
||||||
backend,
|
backend,
|
||||||
|
|||||||
Reference in New Issue
Block a user