[ROCm][MLA] Fuse MLA q/kv RMSNorm + FP8 per-token quant in the FP8 attention path (#44977)
Signed-off-by: Xavier Aguilar <xavier.aguilarfruto@amd.com> Signed-off-by: Xavier Aguilar <Xavier.AguilarFruto@amd.com>
This commit is contained in:
@@ -922,11 +922,145 @@ class MLADualRMSNormPattern(
|
||||
return _replacement
|
||||
|
||||
|
||||
class MLADualRMSPerTokenQuantPattern(
|
||||
VllmPatternReplacement[
|
||||
...,
|
||||
tuple[
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
],
|
||||
]
|
||||
):
|
||||
"""
|
||||
Fuse the MLA FP8 attention path -- q-latent RMSNorm + FP8 *per-token* quant
|
||||
plus kv-latent RMSNorm -- into AITER's ``fused_qk_rmsnorm_per_token_quant``.
|
||||
|
||||
With a per-token FP8 ``q_b_proj`` (Quark / ModelOpt), the earlier
|
||||
``RocmAiterRMSNormQuantFusionPass`` folds the q side into
|
||||
``rocm_aiter_rmsnorm_fused_dynamic_quant`` and leaves the kv side a plain
|
||||
``vllm_ir.rms_norm``. This pattern matches that asymmetric pair::
|
||||
|
||||
gemm -> split_with_sizes([q_dim, kv_dim])
|
||||
+-- q_c -> rocm_aiter_rmsnorm_fused_dynamic_quant -> (q_fp8, q_scale)
|
||||
+-- kv_lora -> split_with_sizes([kv_c_dim, k_pe_dim])
|
||||
+-- kv_c -> vllm_ir.rms_norm -> kv_normed (bf16)
|
||||
+-- k_pe
|
||||
"""
|
||||
|
||||
DYNAMIC_QUANT_OP = rocm_aiter_ops.get_rmsnorm_fused_dynamic_quant_op()
|
||||
FUSED_OP = rocm_aiter_ops.get_fused_mla_dual_rms_norm_per_token_quant_op()
|
||||
|
||||
def __init__(self, epsilon: float) -> None:
|
||||
self._epsilon = epsilon
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
q_dim, kv_c_dim, k_pe_dim = 256, 128, 64
|
||||
return [
|
||||
self.empty_bf16(5, q_dim + kv_c_dim + k_pe_dim),
|
||||
self.empty_bf16(q_dim),
|
||||
self.empty_bf16(kv_c_dim),
|
||||
]
|
||||
|
||||
@property
|
||||
def pattern(
|
||||
self,
|
||||
) -> Callable[
|
||||
...,
|
||||
tuple[
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
],
|
||||
]:
|
||||
eps = self._epsilon
|
||||
dynamic_quant_op = self.DYNAMIC_QUANT_OP
|
||||
|
||||
def _pattern(
|
||||
projected: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
kv_weight: torch.Tensor,
|
||||
) -> tuple[
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
]:
|
||||
q_dim = q_weight.shape[0]
|
||||
kv_dim = projected.shape[-1] - q_dim
|
||||
kv_c_dim = kv_weight.shape[0]
|
||||
k_pe_dim = kv_dim - kv_c_dim
|
||||
q_c, kv_lora = projected.split([q_dim, kv_dim], dim=-1)
|
||||
kv_c, k_pe = kv_lora.split([kv_c_dim, k_pe_dim], dim=-1)
|
||||
q_quant = dynamic_quant_op(
|
||||
x=q_c,
|
||||
weight=q_weight,
|
||||
epsilon=eps,
|
||||
quant_dtype=FP8_DTYPE,
|
||||
)
|
||||
kv_normed = vllm.ir.ops.rms_norm(kv_c, kv_weight, eps)
|
||||
return q_quant[0], q_quant[1], kv_normed, k_pe
|
||||
|
||||
return _pattern
|
||||
|
||||
@property
|
||||
def replacement(
|
||||
self,
|
||||
) -> Callable[
|
||||
...,
|
||||
tuple[
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
],
|
||||
]:
|
||||
eps = self._epsilon
|
||||
fused_op = self.FUSED_OP
|
||||
|
||||
def _replacement(
|
||||
projected: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
kv_weight: torch.Tensor,
|
||||
) -> tuple[
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
]:
|
||||
q_dim = q_weight.shape[0]
|
||||
kv_dim = projected.shape[-1] - q_dim
|
||||
kv_c_dim = kv_weight.shape[0]
|
||||
k_pe_dim = kv_dim - kv_c_dim
|
||||
q_c, kv_lora = projected.split([q_dim, kv_dim], dim=-1)
|
||||
kv_c, k_pe = kv_lora.split([kv_c_dim, k_pe_dim], dim=-1)
|
||||
at = fused_op(
|
||||
q_c,
|
||||
q_weight,
|
||||
kv_c,
|
||||
kv_weight,
|
||||
eps,
|
||||
eps,
|
||||
)
|
||||
# q_fp8, q_scale, kv_normed, k_pe
|
||||
return at[0], at[1], at[2], k_pe
|
||||
|
||||
return _replacement
|
||||
|
||||
|
||||
class MLADualRMSNormFusionPass(VllmFusionPatternMatcherPass):
|
||||
"""
|
||||
Post-grad PatternMatcher pass that fuses paired q / kv RMS norms in
|
||||
MLA attention into ``fused_mla_dual_rms_norm`` backed by aiter's
|
||||
``fused_qk_rmsnorm`` HIP kernel.
|
||||
|
||||
The FP8 attention path is also handled via
|
||||
:class:`MLADualRMSPerTokenQuantPattern`, which fuses the q-latent RMSNorm +
|
||||
FP8 per-token quant together with the kv-latent RMSNorm into
|
||||
``fused_mla_dual_rms_norm_per_token_quant`` backed by aiter's
|
||||
``fused_qk_rmsnorm_per_token_quant`` HIP kernel.
|
||||
"""
|
||||
|
||||
def __init__(self, config: VllmConfig) -> None:
|
||||
@@ -934,3 +1068,4 @@ class MLADualRMSNormFusionPass(VllmFusionPatternMatcherPass):
|
||||
|
||||
for epsilon in [1e-5, 1e-6]:
|
||||
self.register(MLADualRMSNormPattern(epsilon))
|
||||
self.register(MLADualRMSPerTokenQuantPattern(epsilon))
|
||||
|
||||
Reference in New Issue
Block a user