From 88e1b113131903936c4ee333a21d77deaacd50ee Mon Sep 17 00:00:00 2001 From: Jee Jee Li Date: Wed, 26 Aug 2026 09:46:56 +0800 Subject: [PATCH] [Perf] Tune FlashInfer all-reduce thresholds for single-node TP8 on SM103 (#53606) Signed-off-by: Jee Jee Li --- tests/distributed/test_comm_ops.py | 2 +- vllm/compilation/passes/fusion/allreduce_rms_fusion.py | 2 +- vllm/distributed/device_communicators/all_reduce_utils.py | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/distributed/test_comm_ops.py b/tests/distributed/test_comm_ops.py index deac28cde1..5144a1a4cb 100644 --- a/tests/distributed/test_comm_ops.py +++ b/tests/distributed/test_comm_ops.py @@ -326,7 +326,7 @@ def test_cuda_communicator_checkpoints_flashinfer_workspaces( ("mnnvl", 103, 16, 4, 8 * flashinfer_all_reduce.MiB - 1), ("mnnvl", 103, 2, 1, None), ("mnnvl", 103, 12, 3, None), - ("mnnvl", 103, 8, 1, None), + ("mnnvl", 103, 8, 1, 64 * flashinfer_all_reduce.MiB - 1), ("trtllm", 103, 8, 2, None), ("mnnvl", 90, 8, 2, None), ], diff --git a/vllm/compilation/passes/fusion/allreduce_rms_fusion.py b/vllm/compilation/passes/fusion/allreduce_rms_fusion.py index 6650a56de1..559a99ddd9 100644 --- a/vllm/compilation/passes/fusion/allreduce_rms_fusion.py +++ b/vllm/compilation/passes/fusion/allreduce_rms_fusion.py @@ -116,7 +116,7 @@ FI_ALLREDUCE_FUSION_MAX_SIZE_MB: dict[int, dict[int, float]] = { 103: { 2: 64, # 64MB 4: 64, # 64MB - 8: 2, # 2MB + 8: 4, # 4MB 16: 64, # 64MB (mnnvl multi-node) }, 107: { diff --git a/vllm/distributed/device_communicators/all_reduce_utils.py b/vllm/distributed/device_communicators/all_reduce_utils.py index b6da29099b..51c28788cc 100644 --- a/vllm/distributed/device_communicators/all_reduce_utils.py +++ b/vllm/distributed/device_communicators/all_reduce_utils.py @@ -86,6 +86,7 @@ SYMM_MEM_ALL_REDUCE_MAX_SIZES = { # The key is (compute capability, world size, node count). FI_MNNVL_ALLREDUCE_MAX_SIZE_MB: dict[tuple[int, int, int], float] = { (103, 4, 1): 80, + (103, 8, 1): 64, (103, 8, 2): 64, (103, 16, 4): 8, }