[ROCm][AITER] Directly Implement AITER Custom All-reduce in CudaCommunicator (#46065)
Signed-off-by: BadrBasowid <badr.basowid@gmail.com>
This commit is contained in:
@@ -19,7 +19,6 @@ from vllm.compilation.passes.fusion.rms_quant_fusion import (
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.utils import Range
|
||||
from vllm.distributed import get_tp_group, tensor_model_parallel_all_reduce
|
||||
from vllm.distributed.device_communicators.custom_all_reduce import CustomAllreduce
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
@@ -1473,39 +1472,23 @@ class RocmAiterAllReduceFusionPass(VllmFusionPatternMatcherPass):
|
||||
)
|
||||
return
|
||||
|
||||
device_comm = get_tp_group().device_communicator
|
||||
if device_comm is None:
|
||||
logger.warning_once("Device communicator is required.")
|
||||
return
|
||||
|
||||
ca_comm = getattr(device_comm, "ca_comm", None)
|
||||
ca_comm = rocm_aiter_ops.get_aiter_allreduce()
|
||||
if ca_comm is None:
|
||||
logger.warning_once("Custom Allreduce is required.")
|
||||
logger.warning_once(
|
||||
"AITER allreduce fusions are disabled "
|
||||
"because AITER Custom All Reduce is not enabled. "
|
||||
"Set VLLM_ROCM_USE_AITER_CUSTOM_AR=1 "
|
||||
"to enable it."
|
||||
)
|
||||
return
|
||||
self.ca_comm = ca_comm
|
||||
|
||||
assert isinstance(ca_comm, CustomAllreduce)
|
||||
|
||||
group = get_tp_group().cpu_group
|
||||
rocm_aiter_ops.initialize_aiter_allreduce(group, self.device)
|
||||
hidden_dim = config.model_config.get_hidden_size()
|
||||
element_size = torch.tensor([], dtype=self.model_dtype).element_size()
|
||||
max_size = rocm_aiter_ops.get_aiter_allreduce_max_size()
|
||||
if max_size is None:
|
||||
logger.warning("AITER allreduce fusion must be initialized")
|
||||
return
|
||||
|
||||
# Aiter's fused_allreduce_rmsnorm kernel dispatches on hidden_dim.
|
||||
# Before aiter v0.1.12 the launcher was template-specialized on HIDDEN_DIM
|
||||
# and silently no-op'd for sizes outside {512, 1024, 2048, 4096}. From v0.1.12
|
||||
# hidden_dim is a runtime argument. Detect the older API via the missing
|
||||
# `_pool` attribute and skip fusion for unsupported sizes.
|
||||
# Ref (old kernel): https://github.com/ROCm/aiter/blob/6a0e7b26ccf33164785531212cc2ec2cde0b9243/csrc/include/custom_all_reduce.cuh#L2590
|
||||
aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
|
||||
max_size = ca_comm.effective_max_size()
|
||||
_AITER_OLD_FUSED_AR_RMS_HIDDEN = (512, 1024, 2048, 4096)
|
||||
if (
|
||||
aiter_ar is not None
|
||||
and not hasattr(aiter_ar, "_pool")
|
||||
not ca_comm.supports_dynamic_hidden_dim
|
||||
and hidden_dim not in _AITER_OLD_FUSED_AR_RMS_HIDDEN
|
||||
):
|
||||
logger.warning_once(
|
||||
@@ -1515,10 +1498,6 @@ class RocmAiterAllReduceFusionPass(VllmFusionPatternMatcherPass):
|
||||
_AITER_OLD_FUSED_AR_RMS_HIDDEN,
|
||||
hidden_dim,
|
||||
)
|
||||
# Tear down aiter's custom-allreduce so its IPC handles don't
|
||||
# race with vllm's ca_comm on the unfused fallback path.
|
||||
with contextlib.suppress(Exception):
|
||||
rocm_aiter_ops.destroy_aiter_allreduce()
|
||||
return
|
||||
|
||||
max_token_num = max_size // (hidden_dim * element_size)
|
||||
@@ -1532,9 +1511,7 @@ class RocmAiterAllReduceFusionPass(VllmFusionPatternMatcherPass):
|
||||
# fall back to the AR+RMS-only fusion paired with PR #41825's
|
||||
# standalone RMS+quant fusion -- still correct, just leaves the
|
||||
# post-AR quant as a standalone kernel.
|
||||
supports_per_group_quant = (
|
||||
rocm_aiter_ops.has_fused_allreduce_rmsnorm_quant_per_group()
|
||||
)
|
||||
supports_per_group_quant = ca_comm.supports_per_group_quant
|
||||
if not supports_per_group_quant:
|
||||
logger.warning_once(
|
||||
"AITER AR+RMS+per-group-FP8-quant fusion disabled: aiter "
|
||||
@@ -1609,9 +1586,3 @@ class RocmAiterAllReduceFusionPass(VllmFusionPatternMatcherPass):
|
||||
logger.debug(
|
||||
"%s Replaced %s patterns", self.__class__.__name__, self.matched_count
|
||||
)
|
||||
|
||||
def __del__(self) -> None:
|
||||
if getattr(self, "disabled", True):
|
||||
return
|
||||
with contextlib.suppress(Exception):
|
||||
rocm_aiter_ops.destroy_aiter_allreduce()
|
||||
|
||||
Reference in New Issue
Block a user