[Perf] Tune FlashInfer all-reduce selection on SM103 (#53318)
Signed-off-by: Summer Yang <girasoleyang@gmail.com>
This commit is contained in:
@@ -82,6 +82,14 @@ SYMM_MEM_ALL_REDUCE_MAX_SIZES = {
|
||||
},
|
||||
}
|
||||
|
||||
# Per-rank input limit for standalone FlashInfer MNNVL all-reduce.
|
||||
# The key is (compute capability, world size, node count).
|
||||
FI_MNNVL_ALLREDUCE_MAX_SIZE_MB: dict[tuple[int, int, int], float] = {
|
||||
(103, 4, 1): 80,
|
||||
(103, 8, 2): 64,
|
||||
(103, 16, 4): 8,
|
||||
}
|
||||
|
||||
# NCCL symmetric memory allreduce configuration based on H100 and GB200 benchmarks.
|
||||
# PyNCCL-symm outperforms custom_AR for small and large tensor sizes,
|
||||
# while custom_AR wins for mid-range sizes.
|
||||
|
||||
@@ -220,15 +220,17 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
depends on the input tensor.
|
||||
"""
|
||||
all_potential_ar_backends = [
|
||||
"FLASHINFER",
|
||||
"NCCL_SYMM_MEM",
|
||||
"QUICK_REDUCE",
|
||||
"FLASHINFER",
|
||||
"AITER_CUSTOM",
|
||||
"CUSTOM",
|
||||
"SYMM_MEM",
|
||||
"PYNCCL",
|
||||
]
|
||||
enabled_ar_backends: list[str] = []
|
||||
if self.fi_ar_comm is not None and not self.fi_ar_comm.disabled:
|
||||
enabled_ar_backends.append("FLASHINFER")
|
||||
# Mirror the static preconditions of `should_nccl_symm_mem_allreduce`:
|
||||
# VLLM_BATCH_INVARIANT off, NCCL symm mem enabled, world_size meets
|
||||
# min_world_size, and world_size either has a tuned entry in
|
||||
@@ -255,8 +257,6 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
enabled_ar_backends.append("NCCL_SYMM_MEM")
|
||||
if self.qr_comm is not None and not self.qr_comm.disabled:
|
||||
enabled_ar_backends.append("QUICK_REDUCE")
|
||||
if self.fi_ar_comm is not None and not self.fi_ar_comm.disabled:
|
||||
enabled_ar_backends.append("FLASHINFER")
|
||||
if self.aiter_ar_comm is not None and not self.aiter_ar_comm.disabled:
|
||||
enabled_ar_backends.append("AITER_CUSTOM")
|
||||
if self.ca_comm is not None and not self.ca_comm.disabled:
|
||||
@@ -276,16 +276,23 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
)
|
||||
|
||||
def all_reduce(self, input_):
|
||||
fi_ar_comm = self.fi_ar_comm
|
||||
use_fi_ar = (
|
||||
fi_ar_comm is not None
|
||||
and not fi_ar_comm.disabled
|
||||
and fi_ar_comm.should_use_fi_ar(input_)
|
||||
)
|
||||
|
||||
# since currently we perform copy input -> symm_input -> out-of-place AR
|
||||
# return symm_output, we don't need to check if input is symmetric
|
||||
if self.pynccl_comm is not None and should_nccl_symm_mem_allreduce(
|
||||
self.pynccl_comm.world_size, input_
|
||||
if (
|
||||
self.pynccl_comm is not None
|
||||
and not use_fi_ar
|
||||
and should_nccl_symm_mem_allreduce(self.pynccl_comm.world_size, input_)
|
||||
):
|
||||
out = torch.ops.vllm.all_reduce_symmetric_with_copy(input_)
|
||||
if out is not None:
|
||||
return out
|
||||
# always try quick reduce first, then flashinfer, then the AITER or vLLM
|
||||
# custom allreduce, and then pynccl. (quick reduce just for ROCM MI3*)
|
||||
qr_comm = self.qr_comm
|
||||
if (
|
||||
qr_comm is not None
|
||||
@@ -295,12 +302,8 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
out = qr_comm.quick_all_reduce(input_)
|
||||
assert out is not None
|
||||
return out
|
||||
fi_ar_comm = self.fi_ar_comm
|
||||
if (
|
||||
fi_ar_comm is not None
|
||||
and not fi_ar_comm.disabled
|
||||
and fi_ar_comm.should_use_fi_ar(input_)
|
||||
):
|
||||
if use_fi_ar:
|
||||
assert fi_ar_comm is not None
|
||||
out = fi_ar_comm.all_reduce(input_)
|
||||
assert out is not None
|
||||
return out
|
||||
|
||||
@@ -14,7 +14,10 @@ from torch.distributed import ProcessGroup
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config.compilation import PassConfig
|
||||
from vllm.distributed.parallel_state import get_node_count
|
||||
from vllm.distributed.device_communicators.all_reduce_utils import (
|
||||
FI_MNNVL_ALLREDUCE_MAX_SIZE_MB,
|
||||
)
|
||||
from vllm.distributed.parallel_state import _node_count, get_node_count
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -23,6 +26,8 @@ logger = init_logger(__name__)
|
||||
# The empirical value for small batch
|
||||
PDL_ADVANCE_LAUNCH_TOKENS = 16
|
||||
|
||||
MiB = 1024 * 1024
|
||||
|
||||
fi_ar_available = False
|
||||
try:
|
||||
import flashinfer.comm as flashinfer_comm # type: ignore[no-redef]
|
||||
@@ -43,6 +48,23 @@ _fi_ar_quant_workspace = None
|
||||
_fi_ar_workspace_groups: dict[int, ProcessGroup] = {}
|
||||
|
||||
|
||||
def _get_tuned_standalone_max_size(
|
||||
world_size: int,
|
||||
backend: str,
|
||||
group: ProcessGroup,
|
||||
) -> int | None:
|
||||
if backend != "mnnvl":
|
||||
return None
|
||||
capability = current_platform.get_device_capability()
|
||||
if capability is None:
|
||||
return None
|
||||
max_size_mb = FI_MNNVL_ALLREDUCE_MAX_SIZE_MB.get(
|
||||
(capability.to_int(), world_size, _node_count(group))
|
||||
)
|
||||
# Tuned cutoffs are exclusive; store the largest accepted size.
|
||||
return None if max_size_mb is None else int(max_size_mb * MiB) - 1
|
||||
|
||||
|
||||
def _create_workspace(
|
||||
backend: str,
|
||||
world_size: int,
|
||||
@@ -165,6 +187,14 @@ def get_fi_ar_workspace(
|
||||
"'trtllm' backend. Please use 'mnnvl' backend instead."
|
||||
)
|
||||
|
||||
if (
|
||||
envs.VLLM_ALLREDUCE_USE_FLASHINFER
|
||||
and (max_size := _get_tuned_standalone_max_size(world_size, backend, group))
|
||||
is not None
|
||||
):
|
||||
element_size = torch.empty((), dtype=dtype, device="cpu").element_size()
|
||||
max_token_num = max(max_token_num, max_size // (hidden_dim * element_size))
|
||||
|
||||
def _get_or_create(be: str):
|
||||
# Reuse the quant workspace if it was already created with the same backend
|
||||
if _fi_ar_quant_workspace is not None and _fi_ar_quant_workspace.backend == be:
|
||||
@@ -346,20 +376,28 @@ class FlashInferAllReduce:
|
||||
if self.world_size == 1:
|
||||
return
|
||||
|
||||
# Use the same threshold as the allreduce-rms fusion pass
|
||||
# TODO: tune the threshold
|
||||
MiB = 1024 * 1024
|
||||
max_workspace_size = PassConfig.default_fi_allreduce_fusion_max_size_mb().get(
|
||||
self.world_size, None
|
||||
default_max_size_mb = PassConfig.default_fi_allreduce_fusion_max_size_mb().get(
|
||||
self.world_size
|
||||
)
|
||||
if not max_workspace_size:
|
||||
if not default_max_size_mb:
|
||||
logger.warning(
|
||||
"FlashInfer All Reduce is disabled because it "
|
||||
"is not supported for world_size=%d.",
|
||||
self.world_size,
|
||||
)
|
||||
return
|
||||
self.max_workspace_size = max_workspace_size * MiB
|
||||
|
||||
backend, _ = _resolve_fi_ar_backend()
|
||||
tuned_max_size = _get_tuned_standalone_max_size(
|
||||
self.world_size,
|
||||
backend,
|
||||
self.group,
|
||||
)
|
||||
self.max_workspace_size = (
|
||||
tuned_max_size
|
||||
if tuned_max_size is not None
|
||||
else int(default_max_size_mb * MiB)
|
||||
)
|
||||
self.max_num_tokens = 0
|
||||
self.disabled = False
|
||||
|
||||
@@ -394,6 +432,16 @@ class FlashInferAllReduce:
|
||||
if len(input_tensor.shape) != 2:
|
||||
return False
|
||||
|
||||
if input_tensor.dtype not in (
|
||||
torch.float16,
|
||||
torch.bfloat16,
|
||||
torch.float32,
|
||||
):
|
||||
return False
|
||||
|
||||
if input_tensor.nbytes > self.max_workspace_size:
|
||||
return False
|
||||
|
||||
num_tokens, hidden_dim = input_tensor.shape
|
||||
if not self.max_num_tokens:
|
||||
element_size = torch.tensor([], dtype=input_tensor.dtype).element_size()
|
||||
|
||||
Reference in New Issue
Block a user