From 32ad1400d7fa08872430de3f1d0f981dc8db5cb6 Mon Sep 17 00:00:00 2001 From: Shanshan Shen <467638484@qq.com> Date: Fri, 28 Aug 2026 00:37:54 +0800 Subject: [PATCH] [ROCm][Perf] Fuse SWA q/kv RMSNorm and q FP8 group quant for DeepSeek-V4 (#53540) Signed-off-by: Shanshan Shen <87969357+shen-shanshan@users.noreply.github.com> --- tests/kernels/core/test_rocm_aiter_ops.py | 77 +++++++++++++++++- vllm/_aiter_ops.py | 94 ++++++++++++++++++++++ vllm/models/deepseek_v4/amd/rocm.py | 96 +++++++++++++++++++++++ vllm/models/deepseek_v4/attention.py | 78 +++++++++++++++--- 4 files changed, 333 insertions(+), 12 deletions(-) diff --git a/tests/kernels/core/test_rocm_aiter_ops.py b/tests/kernels/core/test_rocm_aiter_ops.py index 9a334502c1..c15afd72d7 100644 --- a/tests/kernels/core/test_rocm_aiter_ops.py +++ b/tests/kernels/core/test_rocm_aiter_ops.py @@ -24,7 +24,7 @@ import pytest import torch import vllm._aiter_ops # noqa: F401 - ensure ops are registered -from tests.kernels.utils import bf16_ulp_distance +from tests.kernels.utils import bf16_ulp_distance, fp8_ulp_distance from vllm.platforms import current_platform from vllm.utils.torch_utils import set_random_seed @@ -531,6 +531,81 @@ def test_rocm_aiter_rmsnorm_with_add_fp8_group_quant_residual_accuracy(): ) +@pytest.mark.parametrize("m", [1, 32]) +def test_rocm_aiter_mla_dual_rmsnorm_group_quant_vs_sequential(m): + """Fused q/kv RMSNorm + q group-quant matches rms_norm -> group_fp8_quant. + + M=1 covers the single-token decode case. M=0 is guarded by the caller + (the ROCm DeepSeek-V4 attention checks num_tokens > 0) and the op has + no zero-row guard itself, so it is intentionally not tested here. + """ + require_aiter() + require_fp8() + from vllm._aiter_ops import rocm_aiter_ops + + nq, nk, group_size = 1536, 576, 128 + q = torch.randn(m, nq, dtype=torch.bfloat16) + kv = torch.randn(m, nk, dtype=torch.bfloat16) + q_weight = torch.ones(nq, dtype=torch.bfloat16) + kv_weight = torch.ones(nk, dtype=torch.bfloat16) + eps = 1e-5 + fp8_dtype = current_platform.fp8_dtype() + + fused_q, fused_scales, fused_kv = rocm_aiter_ops.fused_qk_rmsnorm_group_quant( + q, q_weight, eps, kv, kv_weight, eps, group_size=group_size + ) + _assert_quant_shapes(fused_q, fused_scales, m, nq, fp8_dtype, nq // group_size) + assert fused_kv.shape == (m, nk) + assert fused_kv.dtype == kv.dtype + + # q: dequantized comparison against the sequential composition. + normed_q = rocm_aiter_ops.rms_norm(q, q_weight, eps) + ref_q, ref_scales = rocm_aiter_ops.group_fp8_quant(normed_q, group_size) + _assert_rel_error_quality( + _dequantize_grouped(fused_q, fused_scales, group_size), + _dequantize_grouped(ref_q, ref_scales, group_size), + label="mla_dual_rmsnorm_group_quant", + mean_limit=0.05, + preferred_rel=0.05, + max_rel=0.5, + max_fail_rate=0.01, + ) + + # kv: both sides are aiter RMSNorm kernels; allow one bf16 ULP. + ref_kv = rocm_aiter_ops.rms_norm(kv, kv_weight, eps) + max_ulp = int(bf16_ulp_distance(fused_kv, ref_kv).max().item()) + assert max_ulp <= 1, f"fused kv: max ULP {max_ulp} > 1" + + +def test_rocm_aiter_mla_dual_rmsnorm_group_quant_strided_split_view(): + """Split-view inputs, as produced by the DeepSeek-V4 q/kv split, match + contiguous inputs.""" + require_aiter() + require_fp8() + from vllm._aiter_ops import rocm_aiter_ops + + m, nq, nk, group_size = 16, 1536, 576, 128 + qr_kv = torch.randn(m, nq + nk, dtype=torch.bfloat16) + q_view, kv_view = qr_kv.split([nq, nk], dim=-1) + assert not q_view.is_contiguous() + q_weight, kv_weight = torch.ones(nq + nk, dtype=torch.bfloat16).split([nq, nk]) + eps = 1e-5 + + def run(q, kv): + return rocm_aiter_ops.fused_qk_rmsnorm_group_quant( + q, q_weight, eps, kv, kv_weight, eps, group_size=group_size + ) + + out_strided = run(q_view, kv_view) + out_contig = run(q_view.contiguous(), kv_view.contiguous()) + + q_ulp = int(fp8_ulp_distance(out_strided[0], out_contig[0]).max().item()) + kv_ulp = int(bf16_ulp_distance(out_strided[2], out_contig[2]).max().item()) + assert q_ulp <= 1, f"strided q: max fp8 ULP {q_ulp} > 1" + assert kv_ulp <= 1, f"strided kv: max bf16 ULP {kv_ulp} > 1" + torch.testing.assert_close(out_strided[1], out_contig[1]) + + # -- End-to-end inference chain test --------------------------------------- diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index 94ca4f7fe8..4c215e4fe3 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -1455,6 +1455,70 @@ def _fused_mla_dual_rms_norm_per_token_quant_fake( return q_out, q_scale, kv_normed +def _fused_mla_dual_rms_norm_group_quant_impl( + q: torch.Tensor, + q_weight: torch.Tensor, + kv: torch.Tensor, + kv_weight: torch.Tensor, + q_epsilon: float, + kv_epsilon: float, + group_size: int, + transpose_scale: bool, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Fused MLA q/kv RMSNorm (+ FP8 per-1x128 group quant on q) via AITER. + + Backs the ``fused_mla_dual_rms_norm_group_quant`` custom op used by the + DeepSeek-V4 ROCm attention path when the q latent is group-quantized + (``(M, N/128)`` scales) for the block-scaled FP8 wq_b GEMMs. The q latent + is quantized directly from the fp32 norm accumulator, skipping the bf16 + intermediate the standalone quant path would re-read; the kv latent is + RMS-normed to bf16 and consumed by the fused insert kernel unchanged. + """ + from aiter.ops.fused_qk_rmsnorm_group_quant import ( + fused_qk_rmsnorm_group_quant, + ) + + mq, nq = q.shape + q_out = torch.empty((mq, nq), dtype=FP8_DTYPE, device=q.device) + q_scale = torch.empty((mq, nq // group_size), dtype=torch.float32, device=q.device) + kv_normed = torch.empty(kv.shape, dtype=kv.dtype, device=kv.device) + + # q -> RMSNorm + FP8 group quant (q slot); kv -> RMSNorm only (k slot). + # `split` views are accepted directly (unit inner stride); the kernel + # handles strided inputs, matching the aiter op-test usage. + fused_qk_rmsnorm_group_quant( + q_out_quantized=q_out, + q_out_scale=q_scale, + q=q, + q_weight=q_weight, + q_epsilon=q_epsilon, + k_out=kv_normed, + k=kv, + k_weight=kv_weight, + k_epsilon=kv_epsilon, + group_size=group_size, + transpose_scale=transpose_scale, + ) + return q_out, q_scale, kv_normed + + +def _fused_mla_dual_rms_norm_group_quant_fake( + q: torch.Tensor, + q_weight: torch.Tensor, + kv: torch.Tensor, + kv_weight: torch.Tensor, + q_epsilon: float, + kv_epsilon: float, + group_size: int, + transpose_scale: bool, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + mq, nq = q.shape + q_out = torch.empty((mq, nq), dtype=FP8_DTYPE, device=q.device) + q_scale = torch.empty((mq, nq // group_size), dtype=torch.float32, device=q.device) + kv_normed = torch.empty(kv.shape, dtype=kv.dtype, device=kv.device) + return q_out, q_scale, kv_normed + + def _rocm_aiter_gemm_a8wfp4_impl( x: torch.Tensor, w: torch.Tensor, @@ -2277,6 +2341,13 @@ class rocm_aiter_ops: fake_impl=_fused_mla_dual_rms_norm_per_token_quant_fake, ) + direct_register_custom_op( + op_name="fused_mla_dual_rms_norm_group_quant", + op_func=_fused_mla_dual_rms_norm_group_quant_impl, + mutates_args=[], + fake_impl=_fused_mla_dual_rms_norm_group_quant_fake, + ) + _OPS_REGISTERED = True @staticmethod @@ -2340,6 +2411,29 @@ class rocm_aiter_ops: def get_fused_mla_dual_rms_norm_per_token_quant_op() -> OpOverload: return torch.ops.vllm.fused_mla_dual_rms_norm_per_token_quant.default + @staticmethod + def fused_qk_rmsnorm_group_quant( + q: torch.Tensor, + q_weight: torch.Tensor, + q_epsilon: float, + kv: torch.Tensor, + kv_weight: torch.Tensor, + kv_epsilon: float, + group_size: int = 128, + transpose_scale: bool = False, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Fused q/kv RMSNorm + per-1x128 FP8 group quant on q via AITER.""" + return torch.ops.vllm.fused_mla_dual_rms_norm_group_quant( + q, + q_weight, + kv, + kv_weight, + q_epsilon, + kv_epsilon, + group_size, + transpose_scale, + ) + @staticmethod def rms_norm( x: torch.Tensor, diff --git a/vllm/models/deepseek_v4/amd/rocm.py b/vllm/models/deepseek_v4/amd/rocm.py index dc6620e6c8..3c8d23c4b4 100644 --- a/vllm/models/deepseek_v4/amd/rocm.py +++ b/vllm/models/deepseek_v4/amd/rocm.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import functools from dataclasses import dataclass from typing import cast @@ -60,6 +61,36 @@ def _build_indptr_from_lengths(lengths: torch.Tensor) -> torch.Tensor: return indptr +def apply_pre_quantized_block_scaled_mm( + linear: torch.nn.Module, + x_fp8: torch.Tensor, + x_scale: torch.Tensor, +) -> torch.Tensor: + """Block-scaled fp8 GEMM on pre-quantized activations. + + The fused q/kv norm kernel writes fp8 qr + per-1x128 scales; this + drives the linear's block-scaled GEMM directly with them, bypassing + apply_weights which would re-quantize the fp8 input. Only valid for + the wq_b-style column/replicated linears: their output is the local + TP shard, so no all-reduce is needed. + """ + from vllm.model_executor.kernels.linear.scaled_mm.BlockScaledMMLinearKernel import ( + FP8BlockParams, + ) + + params = FP8BlockParams.from_layer(linear) + weight_scale = ( + params.weight_scale + if params.weight_scale_inv is None + else params.weight_scale_inv + ) + kernel = linear.quant_method.fp8_linear + out = kernel.apply_block_scaled_mm( + A=x_fp8, B=params.weight, As=x_scale, Bs=weight_scale + ) + return out.to(dtype=kernel.config.out_dtype) + + # ROCm sparse prefill keeps this dense combine local so AMD-specific SWA changes # do not touch the shared DeepSeek V4 cache utilities. _SPARSE_PREFILL_TOPK_ALIGNMENT = 128 @@ -621,6 +652,71 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention): indexer_weights, _ = indexer.weights_proj(hidden_states) return qr_kv, kv_score, indexer_kv_score, indexer_weights + @functools.cached_property + def _wq_b_uses_aiter_block_scaled(self) -> bool: + """True when both wq_b GEMMs run the aiter block-scaled fp8 kernel. + + Cached: the linear kernels and the aiter env gates are fixed once + the model is built, so this is evaluated at the first forward + only. + + The fused norm+quant path is only valid if the quant and GEMM it + replaces are exactly the aiter ones; otherwise fall back to the + shared path. + """ + from vllm._aiter_ops import rocm_aiter_ops + from vllm.model_executor.kernels.linear.scaled_mm import ( + Fp8BlockScaledMMLinearKernel, + ) + + if not rocm_aiter_ops.is_linear_fp8_enabled(): + return False + + linears = [self.wq_b] + if self.indexer is not None: + linears.append(self.indexer.wq_b) + for linear in linears: + kernel = getattr(getattr(linear, "quant_method", None), "fp8_linear", None) + if not isinstance(kernel, Fp8BlockScaledMMLinearKernel): + return False + return True + + def _split_qkv_and_norm( + self, qr_kv: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor]: + """Fuse q/kv RMSNorm + per-1x128 fp8 q quant into one aiter kernel. + + The shared path norms q and kv in one triton kernel and the wq_b + linears then re-read the bf16 qr to quantize it. The aiter kernel + computes both RMSNorms (fp32 accumulate) and the fp8 group quant + in a single pass, writing fp8 qr + group scales directly; both + wq_b GEMMs (attention and indexer) then consume that pair and + skip their own input quant. kv stays bf16: the fused insert + kernel RoPE/quantizes it itself. Falls back to the shared path + when the aiter linear path is not active. + """ + qr, kv = qr_kv.split([self.q_lora_rank, self.head_dim], dim=-1) + if not ( + qr.dim() == 2 + and qr.shape[0] > 0 + and self.q_lora_rank % 128 == 0 + and self._wq_b_uses_aiter_block_scaled + ): + return super()._split_qkv_and_norm(qr_kv) + + from vllm._aiter_ops import rocm_aiter_ops + + return rocm_aiter_ops.fused_qk_rmsnorm_group_quant( + q=qr, + q_weight=self.q_norm.weight.data, + q_epsilon=self.eps, + kv=kv, + kv_weight=self.kv_norm.weight.data, + kv_epsilon=self.eps, + group_size=128, + transpose_scale=False, + ) + def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: # ROCm BF16 reference wo_a path (inverse RoPE + einsum) + wo_b. z = rocm_inv_rope_einsum( diff --git a/vllm/models/deepseek_v4/attention.py b/vllm/models/deepseek_v4/attention.py index 81897398e3..61bbda38c6 100644 --- a/vllm/models/deepseek_v4/attention.py +++ b/vllm/models/deepseek_v4/attention.py @@ -373,19 +373,13 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): qr_kv, kv_score, indexer_kv_score, indexer_weights = ( self._run_parallel_input_projections(hidden_states) ) - qr, kv = qr_kv.split([self.q_lora_rank, self.head_dim], dim=-1) - qr, kv = fused_q_kv_rmsnorm( - qr, - kv, - self.q_norm.weight.data, - self.kv_norm.weight.data, - self.eps, - ) + qr, qr_scale, kv = self._split_qkv_and_norm(qr_kv) self._prepare_and_attn_fn( hidden_states, qr, kv, + qr_scale, kv_score, indexer_kv_score, indexer_weights, @@ -397,12 +391,32 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): # Inverse-RoPE + wo_a + wo_b output projection (platform-specific). return self._o_proj(o, positions) + def _split_qkv_and_norm( + self, qr_kv: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor]: + """Split the fused q-lora / kv projection and RMSNorm both halves. + + ``qr_scale`` is None on the shared path. The ROCm subclass returns + a pre-quantized fp8 ``qr`` together with its per-1x128 fp32 scales, + which the downstream wq_b projections consume directly. + """ + qr, kv = qr_kv.split([self.q_lora_rank, self.head_dim], dim=-1) + qr, kv = fused_q_kv_rmsnorm( + qr, + kv, + self.q_norm.weight.data, + self.kv_norm.weight.data, + self.eps, + ) + return qr, None, kv + @eager_break_during_capture def _prepare_and_attn_eager( self, hidden_states: torch.Tensor, qr: torch.Tensor, kv: torch.Tensor, + qr_scale: torch.Tensor | None, kv_score: torch.Tensor, indexer_kv_score: torch.Tensor, indexer_weights: torch.Tensor, @@ -418,6 +432,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): hidden_states, qr, kv, + qr_scale, kv_score, indexer_kv_score, indexer_weights, @@ -430,6 +445,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): hidden_states: torch.Tensor, qr: torch.Tensor, kv: torch.Tensor, + qr_scale: torch.Tensor | None, kv_score: torch.Tensor, indexer_kv_score: torch.Tensor, indexer_weights: torch.Tensor, @@ -446,7 +462,9 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): aux_streams = self.aux_stream_list def project_query_and_cache_kv() -> torch.Tensor: - q = self.wq_b(qr).view(-1, self.n_local_heads, self.head_dim) + q = self._wq_b_proj(qr, qr_scale).view( + -1, self.n_local_heads, self.head_dim + ) return self._fused_qnorm_rope_kv_insert(q, kv, positions, attn_metadata) index_q: torch.Tensor | None = None @@ -468,6 +486,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): indexer_weights, positions, self.indexer_rotary_emb, + qr_scale, ), lambda: compressor(kv_score, positions, self.rotary_emb), ], @@ -507,6 +526,24 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): qr_kv, _ = self.fused_wqa_wkv(hidden_states) return qr_kv + def _wq_b_proj( + self, qr: torch.Tensor, qr_scale: torch.Tensor | None = None + ) -> torch.Tensor: + """Project the q-lora input to the query heads. + + ``qr_scale`` is only set by the ROCm fused path, where ``qr`` is + already fp8-quantized with per-1x128 scales and must bypass the + linear's internal re-quantization (apply_weights would quantize + the fp8 input again). + """ + if qr_scale is None: + return self.wq_b(qr) + from vllm.models.deepseek_v4.amd.rocm import ( + apply_pre_quantized_block_scaled_mm, + ) + + return apply_pre_quantized_block_scaled_mm(self.wq_b, qr, qr_scale) + def _run_parallel_input_projections( self, hidden_states: torch.Tensor ) -> tuple[ @@ -881,6 +918,7 @@ class DeepseekV4Indexer(nn.Module): indexer_weights: torch.Tensor, positions: torch.Tensor, rotary_emb: nn.Module, + qr_scale: torch.Tensor | None = None, ) -> tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]: compressor = self.compressor @@ -911,8 +949,7 @@ class DeepseekV4Indexer(nn.Module): return None, None, None def wq_b_and_q_quant(): - # ReplicatedLinear returns (output, bias); bias is None. - q, _ = self.wq_b(qr) + q = self._wq_b_proj(qr, qr_scale) q = q.view(-1, self.n_head, self.head_dim) return fused_indexer_q_rope_quant( positions, @@ -938,3 +975,22 @@ class DeepseekV4Indexer(nn.Module): else: q, q_scale = q_quant, None return q, q_scale, weights + + def _wq_b_proj( + self, qr: torch.Tensor, qr_scale: torch.Tensor | None = None + ) -> torch.Tensor: + """Project the q-lora input to the indexer query heads. + + Same bypass as ``DeepseekV4Attention._wq_b_proj``: on ROCm the fused + norm path hands over pre-quantized fp8 ``qr`` with per-1x128 scales, + so the linear's internal re-quantization must be skipped. + """ + if qr_scale is None: + # ReplicatedLinear returns (output, bias); bias is None. + q, _ = self.wq_b(qr) + return q + from vllm.models.deepseek_v4.amd.rocm import ( + apply_pre_quantized_block_scaled_mm, + ) + + return apply_pre_quantized_block_scaled_mm(self.wq_b, qr, qr_scale)