[Bugfix] Don't sync-police or retry FlashInfer all-reduce workspace creation in eager mode (#58498)
Signed-off-by: khluu <khluu000@gmail.com> Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Lucas Wilkinson
Nick Hill
parent
25624f65cf
commit
764413559a
@@ -389,6 +389,36 @@ def test_flashinfer_standalone_workspace_size(
|
||||
assert create_workspace.call_args.args[3] == expected
|
||||
|
||||
|
||||
def test_flashinfer_workspace_failure_is_not_retried(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Creation is collective; retrying it per all-reduce desyncs ranks."""
|
||||
create_workspace = Mock(return_value=None)
|
||||
monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_workspace", None)
|
||||
monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_quant_workspace", None)
|
||||
monkeypatch.setattr(flashinfer_all_reduce, "_fi_ar_workspace_failed", False)
|
||||
monkeypatch.setattr(
|
||||
flashinfer_all_reduce,
|
||||
"_resolve_fi_ar_backend",
|
||||
Mock(return_value=("mnnvl", True)),
|
||||
)
|
||||
monkeypatch.setattr(flashinfer_all_reduce, "get_node_count", lambda: 1)
|
||||
monkeypatch.setattr(
|
||||
flashinfer_all_reduce, "_get_tuned_standalone_max_size", Mock(return_value=None)
|
||||
)
|
||||
monkeypatch.setattr(flashinfer_all_reduce, "_create_workspace", create_workspace)
|
||||
args = (2, 0, 128, 4096, torch.bfloat16, Mock())
|
||||
|
||||
assert flashinfer_all_reduce.get_fi_ar_workspace(*args) is None
|
||||
assert flashinfer_all_reduce.get_fi_ar_workspace(*args) is None
|
||||
# One mnnvl attempt plus one trtllm fallback, never repeated.
|
||||
assert create_workspace.call_count == 2
|
||||
|
||||
flashinfer_all_reduce.destroy_fi_ar_workspace()
|
||||
flashinfer_all_reduce.get_fi_ar_workspace(*args)
|
||||
assert create_workspace.call_count == 4
|
||||
|
||||
|
||||
def test_flashinfer_all_reduce_precedes_nccl(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
output = torch.empty(2)
|
||||
fi_ar_comm = Mock(disabled=False)
|
||||
|
||||
Reference in New Issue
Block a user