[Bugfix] Don't sync-police or retry FlashInfer all-reduce workspace creation in eager mode (#58498)
Signed-off-by: khluu <khluu000@gmail.com> Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Lucas Wilkinson
Nick Hill
parent
25624f65cf
commit
764413559a
@@ -389,6 +389,36 @@ def test_flashinfer_standalone_workspace_size(
|
||||
assert create_workspace.call_args.args[3] == expected
|
||||
|
||||
|
||||
def test_flashinfer_workspace_failure_is_not_retried(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Creation is collective; retrying it per all-reduce desyncs ranks."""
|
||||
create_workspace = Mock(return_value=None)
|
||||
monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_workspace", None)
|
||||
monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_quant_workspace", None)
|
||||
monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_workspace_failed", False)
|
||||
monkeypatch.setattr(
|
||||
flashinfer_all_reduce,
|
||||
"_resolve_fi_ar_backend",
|
||||
Mock(return_value=("mnnvl", True)),
|
||||
)
|
||||
monkeypatch.setattr(flashinfer_all_reduce, "get_node_count", lambda: 1)
|
||||
monkeypatch.setattr(
|
||||
flashinfer_all_reduce, "_get_tuned_standalone_max_size", Mock(return_value=None)
|
||||
)
|
||||
monkeypatch.setattr(flashinfer_all_reduce, "_create_workspace", create_workspace)
|
||||
args = (2, 0, 128, 4096, torch.bfloat16, Mock())
|
||||
|
||||
assert flashinfer_all_reduce.get_fi_ar_workspace(*args) is None
|
||||
assert flashinfer_all_reduce.get_fi_ar_workspace(*args) is None
|
||||
# One mnnvl attempt plus one trtllm fallback, never repeated.
|
||||
assert create_workspace.call_count == 2
|
||||
|
||||
flashinfer_all_reduce.destroy_fi_ar_workspace()
|
||||
flashinfer_all_reduce.get_fi_ar_workspace(*args)
|
||||
assert create_workspace.call_count == 4
|
||||
|
||||
|
||||
def test_flashinfer_all_reduce_precedes_nccl(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
output = torch.empty(2)
|
||||
fi_ar_comm = Mock(disabled=False)
|
||||
|
||||
@@ -20,6 +20,7 @@ from vllm.distributed.device_communicators.all_reduce_utils import (
|
||||
from vllm.distributed.parallel_state import _node_count, get_node_count
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -45,6 +46,10 @@ _fi_ar_workspace = None
|
||||
# allreduce backend or a fallback backend when the primary workspace is not
|
||||
# available on the current topology.
|
||||
_fi_ar_quant_workspace = None
|
||||
# Creation is collective, so a failed attempt must not be retried on every
|
||||
# all-reduce call.
|
||||
_fi_ar_workspace_failed = False
|
||||
_fi_ar_quant_workspace_failed = False
|
||||
_fi_ar_workspace_groups: dict[int, ProcessGroup] = {}
|
||||
|
||||
|
||||
@@ -79,16 +84,19 @@ def _create_workspace(
|
||||
rng_state = random.getstate()
|
||||
try:
|
||||
random.seed(int.from_bytes(os.urandom(16), byteorder="big"))
|
||||
workspace = flashinfer_comm.create_allreduce_fusion_workspace(
|
||||
backend=backend,
|
||||
world_size=world_size,
|
||||
rank=rank,
|
||||
max_token_num=max_token_num,
|
||||
hidden_dim=hidden_dim,
|
||||
dtype=dtype,
|
||||
comm_backend=comm_backend,
|
||||
group=group,
|
||||
)
|
||||
# Creation may run lazily inside the first sync-checked forward (e.g.
|
||||
# with enforce_eager, which skips warmup).
|
||||
with gpu_sync_allowed(first_only=True):
|
||||
workspace = flashinfer_comm.create_allreduce_fusion_workspace(
|
||||
backend=backend,
|
||||
world_size=world_size,
|
||||
rank=rank,
|
||||
max_token_num=max_token_num,
|
||||
hidden_dim=hidden_dim,
|
||||
dtype=dtype,
|
||||
comm_backend=comm_backend,
|
||||
group=group,
|
||||
)
|
||||
if backend == "mnnvl" and not getattr(workspace, "mc_ptr", 0):
|
||||
workspace.destroy()
|
||||
logger.warning_once(
|
||||
@@ -175,8 +183,8 @@ def get_fi_ar_workspace(
|
||||
for standalone allreduce. Backend is controlled by
|
||||
VLLM_FLASHINFER_ALLREDUCE_BACKEND env var.
|
||||
"""
|
||||
global _fi_ar_workspace
|
||||
if _fi_ar_workspace is not None:
|
||||
global _fi_ar_workspace, _fi_ar_workspace_failed
|
||||
if _fi_ar_workspace is not None or _fi_ar_workspace_failed:
|
||||
return _fi_ar_workspace
|
||||
|
||||
backend, allow_trtllm_fallback = _resolve_fi_ar_backend()
|
||||
@@ -218,6 +226,7 @@ def get_fi_ar_workspace(
|
||||
f"with backend={backend}"
|
||||
)
|
||||
else:
|
||||
_fi_ar_workspace_failed = True
|
||||
logger.error_once(
|
||||
"Failed to initialize FlashInfer Allreduce norm fusion workspace "
|
||||
f"with backend={backend}"
|
||||
@@ -240,8 +249,8 @@ def get_fi_ar_quant_workspace(
|
||||
non-quant fusion. With ``auto`` this prefers mnnvl and falls back to trtllm
|
||||
only on single-node topologies where mnnvl multicast is unavailable.
|
||||
"""
|
||||
global _fi_ar_quant_workspace
|
||||
if _fi_ar_quant_workspace is not None:
|
||||
global _fi_ar_quant_workspace, _fi_ar_quant_workspace_failed
|
||||
if _fi_ar_quant_workspace is not None or _fi_ar_quant_workspace_failed:
|
||||
return _fi_ar_quant_workspace
|
||||
|
||||
backend, allow_trtllm_fallback = _resolve_fi_ar_backend()
|
||||
@@ -291,6 +300,7 @@ def get_fi_ar_quant_workspace(
|
||||
f"fusion workspace with backend={backend}"
|
||||
)
|
||||
else:
|
||||
_fi_ar_quant_workspace_failed = True
|
||||
logger.error_once(
|
||||
"Failed to initialize FlashInfer Allreduce norm quantization "
|
||||
f"fusion workspace with backend={backend}"
|
||||
@@ -304,6 +314,7 @@ _fi_ar_workspace_lock = threading.Lock()
|
||||
|
||||
def destroy_fi_ar_workspace():
|
||||
global _fi_ar_workspace, _fi_ar_quant_workspace
|
||||
global _fi_ar_workspace_failed, _fi_ar_quant_workspace_failed
|
||||
with _fi_ar_workspace_lock:
|
||||
is_alias = _fi_ar_workspace is _fi_ar_quant_workspace
|
||||
|
||||
@@ -313,6 +324,7 @@ def destroy_fi_ar_workspace():
|
||||
_fi_ar_quant_workspace.destroy()
|
||||
|
||||
_fi_ar_workspace = _fi_ar_quant_workspace = None
|
||||
_fi_ar_workspace_failed = _fi_ar_quant_workspace_failed = False
|
||||
_fi_ar_workspace_groups.clear()
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user