From 764413559a1416511002268ac460bbf8e7ddd394 Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Mon, 28 Sep 2026 12:14:21 -0700 Subject: [PATCH] [Bugfix] Don't sync-police or retry FlashInfer all-reduce workspace creation in eager mode (#58498) Signed-off-by: khluu Signed-off-by: Lucas Wilkinson Co-authored-by: Claude Opus 5.5 Co-authored-by: Lucas Wilkinson Co-authored-by: Nick Hill --- tests/distributed/test_comm_ops.py | 30 ++++++++++++++ .../flashinfer_all_reduce.py | 40 ++++++++++++------- 2 files changed, 56 insertions(+), 14 deletions(-) diff --git a/tests/distributed/test_comm_ops.py b/tests/distributed/test_comm_ops.py index f35ba14c1f..25434a51b5 100644 --- a/tests/distributed/test_comm_ops.py +++ b/tests/distributed/test_comm_ops.py @@ -389,6 +389,36 @@ def test_flashinfer_standalone_workspace_size( assert create_workspace.call_args.args[3] == expected +def test_flashinfer_workspace_failure_is_not_retried( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Creation is collective; retrying it per all-reduce desyncs ranks.""" + create_workspace = Mock(return_value=None) + monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_workspace", None) + monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_quant_workspace", None) + monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_workspace_failed", False) + monkeypatch.setattr( + flashinfer_all_reduce, + "_resolve_fi_ar_backend", + Mock(return_value=("mnnvl", True)), + ) + monkeypatch.setattr(flashinfer_all_reduce, "get_node_count", lambda: 1) + monkeypatch.setattr( + flashinfer_all_reduce, "_get_tuned_standalone_max_size", Mock(return_value=None) + ) + monkeypatch.setattr(flashinfer_all_reduce, "_create_workspace", create_workspace) + args = (2, 0, 128, 4096, torch.bfloat16, Mock()) + + assert flashinfer_all_reduce.get_fi_ar_workspace(*args) is None + assert flashinfer_all_reduce.get_fi_ar_workspace(*args) is None + # One mnnvl attempt plus one trtllm fallback, never repeated. + assert create_workspace.call_count == 2 + + flashinfer_all_reduce.destroy_fi_ar_workspace() + flashinfer_all_reduce.get_fi_ar_workspace(*args) + assert create_workspace.call_count == 4 + + def test_flashinfer_all_reduce_precedes_nccl(monkeypatch: pytest.MonkeyPatch) -> None: output = torch.empty(2) fi_ar_comm = Mock(disabled=False) diff --git a/vllm/distributed/device_communicators/flashinfer_all_reduce.py b/vllm/distributed/device_communicators/flashinfer_all_reduce.py index f1644d699e..9bb17e1528 100644 --- a/vllm/distributed/device_communicators/flashinfer_all_reduce.py +++ b/vllm/distributed/device_communicators/flashinfer_all_reduce.py @@ -20,6 +20,7 @@ from vllm.distributed.device_communicators.all_reduce_utils import ( from vllm.distributed.parallel_state import _node_count, get_node_count from vllm.logger import init_logger from vllm.platforms import current_platform +from vllm.utils.gpu_sync_debug import gpu_sync_allowed logger = init_logger(__name__) @@ -45,6 +46,10 @@ _fi_ar_workspace = None # allreduce backend or a fallback backend when the primary workspace is not # available on the current topology. _fi_ar_quant_workspace = None +# Creation is collective, so a failed attempt must not be retried on every +# all-reduce call. +_fi_ar_workspace_failed = False +_fi_ar_quant_workspace_failed = False _fi_ar_workspace_groups: dict[int, ProcessGroup] = {} @@ -79,16 +84,19 @@ def _create_workspace( rng_state = random.getstate() try: random.seed(int.from_bytes(os.urandom(16), byteorder="big")) - workspace = flashinfer_comm.create_allreduce_fusion_workspace( - backend=backend, - world_size=world_size, - rank=rank, - max_token_num=max_token_num, - hidden_dim=hidden_dim, - dtype=dtype, - comm_backend=comm_backend, - group=group, - ) + # Creation may run lazily inside the first sync-checked forward (e.g. + # with enforce_eager, which skips warmup). + with gpu_sync_allowed(first_only=True): + workspace = flashinfer_comm.create_allreduce_fusion_workspace( + backend=backend, + world_size=world_size, + rank=rank, + max_token_num=max_token_num, + hidden_dim=hidden_dim, + dtype=dtype, + comm_backend=comm_backend, + group=group, + ) if backend == "mnnvl" and not getattr(workspace, "mc_ptr", 0): workspace.destroy() logger.warning_once( @@ -175,8 +183,8 @@ def get_fi_ar_workspace( for standalone allreduce. Backend is controlled by VLLM_FLASHINFER_ALLREDUCE_BACKEND env var. """ - global _fi_ar_workspace - if _fi_ar_workspace is not None: + global _fi_ar_workspace, _fi_ar_workspace_failed + if _fi_ar_workspace is not None or _fi_ar_workspace_failed: return _fi_ar_workspace backend, allow_trtllm_fallback = _resolve_fi_ar_backend() @@ -218,6 +226,7 @@ def get_fi_ar_workspace( f"with backend={backend}" ) else: + _fi_ar_workspace_failed = True logger.error_once( "Failed to initialize FlashInfer Allreduce norm fusion workspace " f"with backend={backend}" @@ -240,8 +249,8 @@ def get_fi_ar_quant_workspace( non-quant fusion. With ``auto`` this prefers mnnvl and falls back to trtllm only on single-node topologies where mnnvl multicast is unavailable. """ - global _fi_ar_quant_workspace - if _fi_ar_quant_workspace is not None: + global _fi_ar_quant_workspace, _fi_ar_quant_workspace_failed + if _fi_ar_quant_workspace is not None or _fi_ar_quant_workspace_failed: return _fi_ar_quant_workspace backend, allow_trtllm_fallback = _resolve_fi_ar_backend() @@ -291,6 +300,7 @@ def get_fi_ar_quant_workspace( f"fusion workspace with backend={backend}" ) else: + _fi_ar_quant_workspace_failed = True logger.error_once( "Failed to initialize FlashInfer Allreduce norm quantization " f"fusion workspace with backend={backend}" @@ -304,6 +314,7 @@ _fi_ar_workspace_lock = threading.Lock() def destroy_fi_ar_workspace(): global _fi_ar_workspace, _fi_ar_quant_workspace + global _fi_ar_workspace_failed, _fi_ar_quant_workspace_failed with _fi_ar_workspace_lock: is_alias = _fi_ar_workspace is _fi_ar_quant_workspace @@ -313,6 +324,7 @@ def destroy_fi_ar_workspace(): _fi_ar_quant_workspace.destroy() _fi_ar_workspace = _fi_ar_quant_workspace = None + _fi_ar_workspace_failed = _fi_ar_quant_workspace_failed = False _fi_ar_workspace_groups.clear()