[ROCm][FEAT] AITER Fused Allreduce + RMSNorm (#37646)
Signed-off-by: vllmellm <vllm.ellm@embeddedllm.com> Signed-off-by: Rita Brugarolas Brufau <rita.brugarolasbrufau@amd.com> Signed-off-by: junkang1991 <junkangchow@gmail.com> Co-authored-by: Rita Brugarolas <Rita.BrugarolasBrufau@amd.com> Co-authored-by: junkang1991 <junkangchow@gmail.com> Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com> Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
This commit is contained in:
co-authored by
Rita Brugarolas
junkang1991
Luka Govedič
TJian
parent
bc635fad23
commit
529c671e80
@@ -190,6 +190,7 @@ class SiluMulBlockQuantPattern(ActivationQuantPattern):
|
||||
is_scale_transposed: bool = False,
|
||||
is_e8m0: bool = False,
|
||||
is_tma_aligned: bool = False,
|
||||
match_aiter: bool = False,
|
||||
) -> None:
|
||||
super().__init__(quant_key)
|
||||
self.quant_matcher = MatcherQuantFP8(
|
||||
|
||||
@@ -12,12 +12,14 @@ from torch._higher_order_ops.auto_functionalize import auto_functionalized
|
||||
from torch._inductor.pattern_matcher import PatternMatcherPass
|
||||
|
||||
import vllm.ir.ops
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.passes.fusion.rms_quant_fusion import (
|
||||
_rms_input_weight_dtype_match,
|
||||
)
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.utils import Range
|
||||
from vllm.distributed import get_tp_group, tensor_model_parallel_all_reduce
|
||||
from vllm.distributed.device_communicators.custom_all_reduce import CustomAllreduce
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
@@ -32,7 +34,12 @@ from vllm.utils.torch_utils import (
|
||||
)
|
||||
|
||||
from ..inductor_pass import enable_fake_mode
|
||||
from ..vllm_inductor_pass import VllmInductorPass, VllmPatternMatcherPass
|
||||
from ..vllm_inductor_pass import (
|
||||
VllmFusionPatternMatcherPass,
|
||||
VllmInductorPass,
|
||||
VllmPatternMatcherPass,
|
||||
VllmPatternReplacement,
|
||||
)
|
||||
from .matcher_utils import MatcherFusedAddRMSNorm, MatcherQuantFP8
|
||||
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
@@ -889,3 +896,204 @@ class AllReduceFusionPass(VllmPatternMatcherPass):
|
||||
return
|
||||
with contextlib.suppress(Exception):
|
||||
destroy_fi_ar_workspace()
|
||||
|
||||
|
||||
# TODO: make BasePattern to inherit from VllmPatternReplacement
|
||||
class AiterAllreduceFusedRMSNormPattern(BasePattern, VllmPatternReplacement):
|
||||
def __init__(
|
||||
self,
|
||||
epsilon: float,
|
||||
dtype: torch.dtype,
|
||||
device: str | None,
|
||||
use_aiter_rmsnorm: bool = True,
|
||||
) -> None:
|
||||
super().__init__(dtype, device)
|
||||
self.dtype = dtype
|
||||
self.epsilon = epsilon
|
||||
self.FUSED_AR_RMSNORM_OP = rocm_aiter_ops.get_fused_allreduce_rmsnorm_op()
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
return [self.empty(5, 16), self.empty(16)]
|
||||
|
||||
@property
|
||||
def pattern(self):
|
||||
def _pattern(
|
||||
input: torch.Tensor, weight: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
allreduce_output = tensor_model_parallel_all_reduce(input)
|
||||
rms = vllm.ir.ops.rms_norm(allreduce_output, weight, self.epsilon)
|
||||
|
||||
return rms, allreduce_output
|
||||
|
||||
return _pattern
|
||||
|
||||
@property
|
||||
def replacement(self):
|
||||
def _replacement(
|
||||
input: torch.Tensor, weight: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
residual = torch.empty_like(input)
|
||||
allreduce = self.FUSED_AR_RMSNORM_OP(
|
||||
input_=input,
|
||||
residual=residual,
|
||||
weight=weight,
|
||||
epsilon=self.epsilon,
|
||||
)
|
||||
return allreduce[0], allreduce[1]
|
||||
|
||||
return _replacement
|
||||
|
||||
|
||||
class AiterAllreduceFusedAddRMSNormPattern(BasePattern, VllmPatternReplacement):
|
||||
def __init__(
|
||||
self,
|
||||
epsilon: float,
|
||||
dtype: torch.dtype,
|
||||
device: str | None,
|
||||
use_aiter_rmsnorm: bool = True,
|
||||
) -> None:
|
||||
super().__init__(dtype, device)
|
||||
self.epsilon = epsilon
|
||||
self.dtype = dtype
|
||||
self.rmsnorm_matcher = MatcherFusedAddRMSNorm(
|
||||
epsilon, match_rocm_aiter=use_aiter_rmsnorm
|
||||
)
|
||||
self.FUSED_AR_RMSNORM_OP = rocm_aiter_ops.get_fused_allreduce_rmsnorm_op()
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
input, residual, weight = self.rmsnorm_matcher.inputs()
|
||||
|
||||
return [residual, input.to(self.dtype), weight]
|
||||
|
||||
@property
|
||||
def pattern(self):
|
||||
def _pattern(
|
||||
residual: torch.Tensor, input: torch.Tensor, weight: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
allreduce_output = tensor_model_parallel_all_reduce(input)
|
||||
rms, residual = self.rmsnorm_matcher(allreduce_output, weight, residual)
|
||||
|
||||
return rms, residual
|
||||
|
||||
return _pattern
|
||||
|
||||
@property
|
||||
def replacement(self):
|
||||
def _replacement(
|
||||
residual: torch.Tensor, input: torch.Tensor, weight: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
allreduce = self.FUSED_AR_RMSNORM_OP(
|
||||
input_=input,
|
||||
residual=residual,
|
||||
weight=weight,
|
||||
epsilon=self.epsilon,
|
||||
)
|
||||
return allreduce[0], allreduce[1]
|
||||
|
||||
return _replacement
|
||||
|
||||
|
||||
class RocmAiterAllReduceFusionPass(VllmFusionPatternMatcherPass):
|
||||
def __init__(self, config: VllmConfig) -> None:
|
||||
super().__init__(config, "rocm_aiter_allreduce_fusion_pass")
|
||||
self.disabled = True
|
||||
self.tp_size = get_tensor_model_parallel_world_size()
|
||||
if self.tp_size <= 1:
|
||||
logger.warning_once("AllReduce fusion pass is disabled for tp_size <= 1.")
|
||||
return
|
||||
|
||||
if config.model_config is None:
|
||||
logger.warning_once(
|
||||
"AllReduce fusion pass is disabled for missing model_config."
|
||||
)
|
||||
return
|
||||
|
||||
device_comm = get_tp_group().device_communicator
|
||||
if device_comm is None:
|
||||
logger.warning_once("Device communicator is required.")
|
||||
return
|
||||
|
||||
ca_comm = getattr(device_comm, "ca_comm", None)
|
||||
if ca_comm is None:
|
||||
logger.warning_once("Custom Allreduce is required.")
|
||||
return
|
||||
self.ca_comm = ca_comm
|
||||
|
||||
assert isinstance(ca_comm, CustomAllreduce)
|
||||
|
||||
group = get_tp_group().cpu_group
|
||||
rocm_aiter_ops.initialize_aiter_allreduce(group, self.device)
|
||||
hidden_dim = config.model_config.get_hidden_size()
|
||||
element_size = torch.tensor([], dtype=self.model_dtype).element_size()
|
||||
max_size = rocm_aiter_ops.get_aiter_allreduce_max_size()
|
||||
if max_size is None:
|
||||
logger.warning("AITER allreduce fusion must be initialized")
|
||||
return
|
||||
|
||||
# Aiter's fused_allreduce_rmsnorm kernel dispatches on hidden_dim.
|
||||
# Before aiter v0.1.12 the launcher was template-specialized on HIDDEN_DIM
|
||||
# and silently no-op'd for sizes outside {512, 1024, 2048, 4096}. From v0.1.12
|
||||
# hidden_dim is a runtime argument. Detect the older API via the missing
|
||||
# `_pool` attribute and skip fusion for unsupported sizes.
|
||||
# Ref (old kernel): https://github.com/ROCm/aiter/blob/6a0e7b26ccf33164785531212cc2ec2cde0b9243/csrc/include/custom_all_reduce.cuh#L2590
|
||||
aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
|
||||
_AITER_OLD_FUSED_AR_RMS_HIDDEN = (512, 1024, 2048, 4096)
|
||||
if (
|
||||
aiter_ar is not None
|
||||
and not hasattr(aiter_ar, "_pool")
|
||||
and hidden_dim not in _AITER_OLD_FUSED_AR_RMS_HIDDEN
|
||||
):
|
||||
logger.warning_once(
|
||||
"AITER allreduce-rmsnorm fusion disabled: aiter<0.1.12 "
|
||||
"only supports hidden_dim in %s; got %d. Upgrade aiter to "
|
||||
">=0.1.12 to enable fusion for this model.",
|
||||
_AITER_OLD_FUSED_AR_RMS_HIDDEN,
|
||||
hidden_dim,
|
||||
)
|
||||
# Tear down aiter's custom-allreduce so its IPC handles don't
|
||||
# race with vllm's ca_comm on the unfused fallback path.
|
||||
with contextlib.suppress(Exception):
|
||||
rocm_aiter_ops.destroy_aiter_allreduce()
|
||||
return
|
||||
|
||||
max_token_num = max_size // (hidden_dim * element_size)
|
||||
self.max_token_num = min(
|
||||
max_token_num,
|
||||
config.scheduler_config.max_num_batched_tokens,
|
||||
)
|
||||
|
||||
for epsilon in [1e-5, 1e-6]:
|
||||
self.register(
|
||||
AiterAllreduceFusedRMSNormPattern(
|
||||
epsilon,
|
||||
self.model_dtype,
|
||||
self.device,
|
||||
)
|
||||
)
|
||||
self.register(
|
||||
AiterAllreduceFusedAddRMSNormPattern(
|
||||
epsilon,
|
||||
self.model_dtype,
|
||||
self.device,
|
||||
)
|
||||
)
|
||||
|
||||
# WARNING: This is a hack to clear the pattern matcher cache
|
||||
# and allow multiple values of epsilon.
|
||||
torch._inductor.pattern_matcher._seen_patterns.clear()
|
||||
|
||||
self.disabled = False
|
||||
|
||||
self.dump_patterns(config, self.pm_pass)
|
||||
|
||||
def is_applicable_for_range(self, compile_range: Range) -> bool:
|
||||
if self.disabled:
|
||||
logger.warning_once("AllReduce fusion pass is disabled.")
|
||||
return False
|
||||
return bool(compile_range.end <= self.max_token_num)
|
||||
|
||||
def __del__(self) -> None:
|
||||
if getattr(self, "disabled", True):
|
||||
return
|
||||
with contextlib.suppress(Exception):
|
||||
rocm_aiter_ops.destroy_aiter_allreduce()
|
||||
|
||||
@@ -18,6 +18,9 @@ from .ir.lowering_pass import VllmIRLoweringPass
|
||||
from .vllm_inductor_pass import VllmInductorPass, VllmPatternMatcherPass
|
||||
|
||||
if rocm_aiter_ops.is_enabled():
|
||||
from .fusion.allreduce_rms_fusion import (
|
||||
RocmAiterAllReduceFusionPass,
|
||||
)
|
||||
from .fusion.rocm_aiter_fusion import (
|
||||
MLADualRMSNormFusionPass,
|
||||
RocmAiterRMSNormQuantFusionPass,
|
||||
@@ -137,7 +140,10 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
|
||||
self.passes += [AsyncTPPass(config)]
|
||||
|
||||
if self.pass_config.fuse_allreduce_rms:
|
||||
self.passes += [AllReduceFusionPass(config)]
|
||||
if rocm_aiter_ops.is_enabled():
|
||||
self.passes += [RocmAiterAllReduceFusionPass(config)]
|
||||
else:
|
||||
self.passes += [AllReduceFusionPass(config)]
|
||||
|
||||
if self.pass_config.fuse_minimax_qk_norm:
|
||||
self.passes += [MiniMaxQKNormPass(config)]
|
||||
|
||||
Reference in New Issue
Block a user