[Bugfix] Don't sync-police or retry FlashInfer all-reduce workspace creation in eager mode (#58498)

Signed-off-by: khluu <khluu000@gmail.com>
Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Kevin H. Luu
2026-09-28 12:14:21 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 Lucas Wilkinson Nick Hill
parent 25624f65cf
commit 764413559a
2 changed files with 56 additions and 14 deletions
+30
View File
@@ -389,6 +389,36 @@ def test_flashinfer_standalone_workspace_size(
assert create_workspace.call_args.args[3] == expected
def test_flashinfer_workspace_failure_is_not_retried(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Creation is collective; retrying it per all-reduce desyncs ranks."""
create_workspace = Mock(return_value=None)
monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_workspace", None)
monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_quant_workspace", None)
monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_workspace_failed", False)
monkeypatch.setattr(
flashinfer_all_reduce,
"_resolve_fi_ar_backend",
Mock(return_value=("mnnvl", True)),
)
monkeypatch.setattr(flashinfer_all_reduce, "get_node_count", lambda: 1)
monkeypatch.setattr(
flashinfer_all_reduce, "_get_tuned_standalone_max_size", Mock(return_value=None)
)
monkeypatch.setattr(flashinfer_all_reduce, "_create_workspace", create_workspace)
args = (2, 0, 128, 4096, torch.bfloat16, Mock())
assert flashinfer_all_reduce.get_fi_ar_workspace(*args) is None
assert flashinfer_all_reduce.get_fi_ar_workspace(*args) is None
# One mnnvl attempt plus one trtllm fallback, never repeated.
assert create_workspace.call_count == 2
flashinfer_all_reduce.destroy_fi_ar_workspace()
flashinfer_all_reduce.get_fi_ar_workspace(*args)
assert create_workspace.call_count == 4
def test_flashinfer_all_reduce_precedes_nccl(monkeypatch: pytest.MonkeyPatch) -> None:
output = torch.empty(2)
fi_ar_comm = Mock(disabled=False)
@@ -20,6 +20,7 @@ from vllm.distributed.device_communicators.all_reduce_utils import (
from vllm.distributed.parallel_state import _node_count, get_node_count
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
logger = init_logger(__name__)
@@ -45,6 +46,10 @@ _fi_ar_workspace = None
# allreduce backend or a fallback backend when the primary workspace is not
# available on the current topology.
_fi_ar_quant_workspace = None
# Creation is collective, so a failed attempt must not be retried on every
# all-reduce call.
_fi_ar_workspace_failed = False
_fi_ar_quant_workspace_failed = False
_fi_ar_workspace_groups: dict[int, ProcessGroup] = {}
@@ -79,16 +84,19 @@ def _create_workspace(
rng_state = random.getstate()
try:
random.seed(int.from_bytes(os.urandom(16), byteorder="big"))
workspace = flashinfer_comm.create_allreduce_fusion_workspace(
backend=backend,
world_size=world_size,
rank=rank,
max_token_num=max_token_num,
hidden_dim=hidden_dim,
dtype=dtype,
comm_backend=comm_backend,
group=group,
)
# Creation may run lazily inside the first sync-checked forward (e.g.
# with enforce_eager, which skips warmup).
with gpu_sync_allowed(first_only=True):
workspace = flashinfer_comm.create_allreduce_fusion_workspace(
backend=backend,
world_size=world_size,
rank=rank,
max_token_num=max_token_num,
hidden_dim=hidden_dim,
dtype=dtype,
comm_backend=comm_backend,
group=group,
)
if backend == "mnnvl" and not getattr(workspace, "mc_ptr", 0):
workspace.destroy()
logger.warning_once(
@@ -175,8 +183,8 @@ def get_fi_ar_workspace(
for standalone allreduce. Backend is controlled by
VLLM_FLASHINFER_ALLREDUCE_BACKEND env var.
"""
global _fi_ar_workspace
if _fi_ar_workspace is not None:
global _fi_ar_workspace, _fi_ar_workspace_failed
if _fi_ar_workspace is not None or _fi_ar_workspace_failed:
return _fi_ar_workspace
backend, allow_trtllm_fallback = _resolve_fi_ar_backend()
@@ -218,6 +226,7 @@ def get_fi_ar_workspace(
f"with backend={backend}"
)
else:
_fi_ar_workspace_failed = True
logger.error_once(
"Failed to initialize FlashInfer Allreduce norm fusion workspace "
f"with backend={backend}"
@@ -240,8 +249,8 @@ def get_fi_ar_quant_workspace(
non-quant fusion. With ``auto`` this prefers mnnvl and falls back to trtllm
only on single-node topologies where mnnvl multicast is unavailable.
"""
global _fi_ar_quant_workspace
if _fi_ar_quant_workspace is not None:
global _fi_ar_quant_workspace, _fi_ar_quant_workspace_failed
if _fi_ar_quant_workspace is not None or _fi_ar_quant_workspace_failed:
return _fi_ar_quant_workspace
backend, allow_trtllm_fallback = _resolve_fi_ar_backend()
@@ -291,6 +300,7 @@ def get_fi_ar_quant_workspace(
f"fusion workspace with backend={backend}"
)
else:
_fi_ar_quant_workspace_failed = True
logger.error_once(
"Failed to initialize FlashInfer Allreduce norm quantization "
f"fusion workspace with backend={backend}"
@@ -304,6 +314,7 @@ _fi_ar_workspace_lock = threading.Lock()
def destroy_fi_ar_workspace():
global _fi_ar_workspace, _fi_ar_quant_workspace
global _fi_ar_workspace_failed, _fi_ar_quant_workspace_failed
with _fi_ar_workspace_lock:
is_alias = _fi_ar_workspace is _fi_ar_quant_workspace
@@ -313,6 +324,7 @@ def destroy_fi_ar_workspace():
_fi_ar_quant_workspace.destroy()
_fi_ar_workspace = _fi_ar_quant_workspace = None
_fi_ar_workspace_failed = _fi_ar_quant_workspace_failed = False
_fi_ar_workspace_groups.clear()