[ROCm][Perf] Fuse SWA q/kv RMSNorm and q FP8 group quant for DeepSeek-V4 (#53540)

Signed-off-by: Shanshan Shen <87969357+shen-shanshan@users.noreply.github.com>
This commit is contained in:
Shanshan Shen
2026-08-27 09:37:54 -07:00
committed by GitHub
parent 9650dc73e2
commit 32ad1400d7
4 changed files with 333 additions and 12 deletions
+76 -1
View File
@@ -24,7 +24,7 @@ import pytest
import torch
import vllm._aiter_ops # noqa: F401 - ensure ops are registered
from tests.kernels.utils import bf16_ulp_distance
from tests.kernels.utils import bf16_ulp_distance, fp8_ulp_distance
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
@@ -531,6 +531,81 @@ def test_rocm_aiter_rmsnorm_with_add_fp8_group_quant_residual_accuracy():
)
@pytest.mark.parametrize("m", [1, 32])
def test_rocm_aiter_mla_dual_rmsnorm_group_quant_vs_sequential(m):
"""Fused q/kv RMSNorm + q group-quant matches rms_norm -> group_fp8_quant.
M=1 covers the single-token decode case. M=0 is guarded by the caller
(the ROCm DeepSeek-V4 attention checks num_tokens > 0) and the op has
no zero-row guard itself, so it is intentionally not tested here.
"""
require_aiter()
require_fp8()
from vllm._aiter_ops import rocm_aiter_ops
nq, nk, group_size = 1536, 576, 128
q = torch.randn(m, nq, dtype=torch.bfloat16)
kv = torch.randn(m, nk, dtype=torch.bfloat16)
q_weight = torch.ones(nq, dtype=torch.bfloat16)
kv_weight = torch.ones(nk, dtype=torch.bfloat16)
eps = 1e-5
fp8_dtype = current_platform.fp8_dtype()
fused_q, fused_scales, fused_kv = rocm_aiter_ops.fused_qk_rmsnorm_group_quant(
q, q_weight, eps, kv, kv_weight, eps, group_size=group_size
)
_assert_quant_shapes(fused_q, fused_scales, m, nq, fp8_dtype, nq // group_size)
assert fused_kv.shape == (m, nk)
assert fused_kv.dtype == kv.dtype
# q: dequantized comparison against the sequential composition.
normed_q = rocm_aiter_ops.rms_norm(q, q_weight, eps)
ref_q, ref_scales = rocm_aiter_ops.group_fp8_quant(normed_q, group_size)
_assert_rel_error_quality(
_dequantize_grouped(fused_q, fused_scales, group_size),
_dequantize_grouped(ref_q, ref_scales, group_size),
label="mla_dual_rmsnorm_group_quant",
mean_limit=0.05,
preferred_rel=0.05,
max_rel=0.5,
max_fail_rate=0.01,
)
# kv: both sides are aiter RMSNorm kernels; allow one bf16 ULP.
ref_kv = rocm_aiter_ops.rms_norm(kv, kv_weight, eps)
max_ulp = int(bf16_ulp_distance(fused_kv, ref_kv).max().item())
assert max_ulp <= 1, f"fused kv: max ULP {max_ulp} > 1"
def test_rocm_aiter_mla_dual_rmsnorm_group_quant_strided_split_view():
"""Split-view inputs, as produced by the DeepSeek-V4 q/kv split, match
contiguous inputs."""
require_aiter()
require_fp8()
from vllm._aiter_ops import rocm_aiter_ops
m, nq, nk, group_size = 16, 1536, 576, 128
qr_kv = torch.randn(m, nq + nk, dtype=torch.bfloat16)
q_view, kv_view = qr_kv.split([nq, nk], dim=-1)
assert not q_view.is_contiguous()
q_weight, kv_weight = torch.ones(nq + nk, dtype=torch.bfloat16).split([nq, nk])
eps = 1e-5
def run(q, kv):
return rocm_aiter_ops.fused_qk_rmsnorm_group_quant(
q, q_weight, eps, kv, kv_weight, eps, group_size=group_size
)
out_strided = run(q_view, kv_view)
out_contig = run(q_view.contiguous(), kv_view.contiguous())
q_ulp = int(fp8_ulp_distance(out_strided[0], out_contig[0]).max().item())
kv_ulp = int(bf16_ulp_distance(out_strided[2], out_contig[2]).max().item())
assert q_ulp <= 1, f"strided q: max fp8 ULP {q_ulp} > 1"
assert kv_ulp <= 1, f"strided kv: max bf16 ULP {kv_ulp} > 1"
torch.testing.assert_close(out_strided[1], out_contig[1])
# -- End-to-end inference chain test ---------------------------------------
+94
View File
@@ -1455,6 +1455,70 @@ def _fused_mla_dual_rms_norm_per_token_quant_fake(
return q_out, q_scale, kv_normed
def _fused_mla_dual_rms_norm_group_quant_impl(
q: torch.Tensor,
q_weight: torch.Tensor,
kv: torch.Tensor,
kv_weight: torch.Tensor,
q_epsilon: float,
kv_epsilon: float,
group_size: int,
transpose_scale: bool,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Fused MLA q/kv RMSNorm (+ FP8 per-1x128 group quant on q) via AITER.
Backs the ``fused_mla_dual_rms_norm_group_quant`` custom op used by the
DeepSeek-V4 ROCm attention path when the q latent is group-quantized
(``(M, N/128)`` scales) for the block-scaled FP8 wq_b GEMMs. The q latent
is quantized directly from the fp32 norm accumulator, skipping the bf16
intermediate the standalone quant path would re-read; the kv latent is
RMS-normed to bf16 and consumed by the fused insert kernel unchanged.
"""
from aiter.ops.fused_qk_rmsnorm_group_quant import (
fused_qk_rmsnorm_group_quant,
)
mq, nq = q.shape
q_out = torch.empty((mq, nq), dtype=FP8_DTYPE, device=q.device)
q_scale = torch.empty((mq, nq // group_size), dtype=torch.float32, device=q.device)
kv_normed = torch.empty(kv.shape, dtype=kv.dtype, device=kv.device)
# q -> RMSNorm + FP8 group quant (q slot); kv -> RMSNorm only (k slot).
# `split` views are accepted directly (unit inner stride); the kernel
# handles strided inputs, matching the aiter op-test usage.
fused_qk_rmsnorm_group_quant(
q_out_quantized=q_out,
q_out_scale=q_scale,
q=q,
q_weight=q_weight,
q_epsilon=q_epsilon,
k_out=kv_normed,
k=kv,
k_weight=kv_weight,
k_epsilon=kv_epsilon,
group_size=group_size,
transpose_scale=transpose_scale,
)
return q_out, q_scale, kv_normed
def _fused_mla_dual_rms_norm_group_quant_fake(
q: torch.Tensor,
q_weight: torch.Tensor,
kv: torch.Tensor,
kv_weight: torch.Tensor,
q_epsilon: float,
kv_epsilon: float,
group_size: int,
transpose_scale: bool,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
mq, nq = q.shape
q_out = torch.empty((mq, nq), dtype=FP8_DTYPE, device=q.device)
q_scale = torch.empty((mq, nq // group_size), dtype=torch.float32, device=q.device)
kv_normed = torch.empty(kv.shape, dtype=kv.dtype, device=kv.device)
return q_out, q_scale, kv_normed
def _rocm_aiter_gemm_a8wfp4_impl(
x: torch.Tensor,
w: torch.Tensor,
@@ -2277,6 +2341,13 @@ class rocm_aiter_ops:
fake_impl=_fused_mla_dual_rms_norm_per_token_quant_fake,
)
direct_register_custom_op(
op_name="fused_mla_dual_rms_norm_group_quant",
op_func=_fused_mla_dual_rms_norm_group_quant_impl,
mutates_args=[],
fake_impl=_fused_mla_dual_rms_norm_group_quant_fake,
)
_OPS_REGISTERED = True
@staticmethod
@@ -2340,6 +2411,29 @@ class rocm_aiter_ops:
def get_fused_mla_dual_rms_norm_per_token_quant_op() -> OpOverload:
return torch.ops.vllm.fused_mla_dual_rms_norm_per_token_quant.default
@staticmethod
def fused_qk_rmsnorm_group_quant(
q: torch.Tensor,
q_weight: torch.Tensor,
q_epsilon: float,
kv: torch.Tensor,
kv_weight: torch.Tensor,
kv_epsilon: float,
group_size: int = 128,
transpose_scale: bool = False,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Fused q/kv RMSNorm + per-1x128 FP8 group quant on q via AITER."""
return torch.ops.vllm.fused_mla_dual_rms_norm_group_quant(
q,
q_weight,
kv,
kv_weight,
q_epsilon,
kv_epsilon,
group_size,
transpose_scale,
)
@staticmethod
def rms_norm(
x: torch.Tensor,
+96
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools
from dataclasses import dataclass
from typing import cast
@@ -60,6 +61,36 @@ def _build_indptr_from_lengths(lengths: torch.Tensor) -> torch.Tensor:
return indptr
def apply_pre_quantized_block_scaled_mm(
linear: torch.nn.Module,
x_fp8: torch.Tensor,
x_scale: torch.Tensor,
) -> torch.Tensor:
"""Block-scaled fp8 GEMM on pre-quantized activations.
The fused q/kv norm kernel writes fp8 qr + per-1x128 scales; this
drives the linear's block-scaled GEMM directly with them, bypassing
apply_weights which would re-quantize the fp8 input. Only valid for
the wq_b-style column/replicated linears: their output is the local
TP shard, so no all-reduce is needed.
"""
from vllm.model_executor.kernels.linear.scaled_mm.BlockScaledMMLinearKernel import (
FP8BlockParams,
)
params = FP8BlockParams.from_layer(linear)
weight_scale = (
params.weight_scale
if params.weight_scale_inv is None
else params.weight_scale_inv
)
kernel = linear.quant_method.fp8_linear
out = kernel.apply_block_scaled_mm(
A=x_fp8, B=params.weight, As=x_scale, Bs=weight_scale
)
return out.to(dtype=kernel.config.out_dtype)
# ROCm sparse prefill keeps this dense combine local so AMD-specific SWA changes
# do not touch the shared DeepSeek V4 cache utilities.
_SPARSE_PREFILL_TOPK_ALIGNMENT = 128
@@ -621,6 +652,71 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
indexer_weights, _ = indexer.weights_proj(hidden_states)
return qr_kv, kv_score, indexer_kv_score, indexer_weights
@functools.cached_property
def _wq_b_uses_aiter_block_scaled(self) -> bool:
"""True when both wq_b GEMMs run the aiter block-scaled fp8 kernel.
Cached: the linear kernels and the aiter env gates are fixed once
the model is built, so this is evaluated at the first forward
only.
The fused norm+quant path is only valid if the quant and GEMM it
replaces are exactly the aiter ones; otherwise fall back to the
shared path.
"""
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.kernels.linear.scaled_mm import (
Fp8BlockScaledMMLinearKernel,
)
if not rocm_aiter_ops.is_linear_fp8_enabled():
return False
linears = [self.wq_b]
if self.indexer is not None:
linears.append(self.indexer.wq_b)
for linear in linears:
kernel = getattr(getattr(linear, "quant_method", None), "fp8_linear", None)
if not isinstance(kernel, Fp8BlockScaledMMLinearKernel):
return False
return True
def _split_qkv_and_norm(
self, qr_kv: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor]:
"""Fuse q/kv RMSNorm + per-1x128 fp8 q quant into one aiter kernel.
The shared path norms q and kv in one triton kernel and the wq_b
linears then re-read the bf16 qr to quantize it. The aiter kernel
computes both RMSNorms (fp32 accumulate) and the fp8 group quant
in a single pass, writing fp8 qr + group scales directly; both
wq_b GEMMs (attention and indexer) then consume that pair and
skip their own input quant. kv stays bf16: the fused insert
kernel RoPE/quantizes it itself. Falls back to the shared path
when the aiter linear path is not active.
"""
qr, kv = qr_kv.split([self.q_lora_rank, self.head_dim], dim=-1)
if not (
qr.dim() == 2
and qr.shape[0] > 0
and self.q_lora_rank % 128 == 0
and self._wq_b_uses_aiter_block_scaled
):
return super()._split_qkv_and_norm(qr_kv)
from vllm._aiter_ops import rocm_aiter_ops
return rocm_aiter_ops.fused_qk_rmsnorm_group_quant(
q=qr,
q_weight=self.q_norm.weight.data,
q_epsilon=self.eps,
kv=kv,
kv_weight=self.kv_norm.weight.data,
kv_epsilon=self.eps,
group_size=128,
transpose_scale=False,
)
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
# ROCm BF16 reference wo_a path (inverse RoPE + einsum) + wo_b.
z = rocm_inv_rope_einsum(
+67 -11
View File
@@ -373,19 +373,13 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
qr_kv, kv_score, indexer_kv_score, indexer_weights = (
self._run_parallel_input_projections(hidden_states)
)
qr, kv = qr_kv.split([self.q_lora_rank, self.head_dim], dim=-1)
qr, kv = fused_q_kv_rmsnorm(
qr,
kv,
self.q_norm.weight.data,
self.kv_norm.weight.data,
self.eps,
)
qr, qr_scale, kv = self._split_qkv_and_norm(qr_kv)
self._prepare_and_attn_fn(
hidden_states,
qr,
kv,
qr_scale,
kv_score,
indexer_kv_score,
indexer_weights,
@@ -397,12 +391,32 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
# Inverse-RoPE + wo_a + wo_b output projection (platform-specific).
return self._o_proj(o, positions)
def _split_qkv_and_norm(
self, qr_kv: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor]:
"""Split the fused q-lora / kv projection and RMSNorm both halves.
``qr_scale`` is None on the shared path. The ROCm subclass returns
a pre-quantized fp8 ``qr`` together with its per-1x128 fp32 scales,
which the downstream wq_b projections consume directly.
"""
qr, kv = qr_kv.split([self.q_lora_rank, self.head_dim], dim=-1)
qr, kv = fused_q_kv_rmsnorm(
qr,
kv,
self.q_norm.weight.data,
self.kv_norm.weight.data,
self.eps,
)
return qr, None, kv
@eager_break_during_capture
def _prepare_and_attn_eager(
self,
hidden_states: torch.Tensor,
qr: torch.Tensor,
kv: torch.Tensor,
qr_scale: torch.Tensor | None,
kv_score: torch.Tensor,
indexer_kv_score: torch.Tensor,
indexer_weights: torch.Tensor,
@@ -418,6 +432,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
hidden_states,
qr,
kv,
qr_scale,
kv_score,
indexer_kv_score,
indexer_weights,
@@ -430,6 +445,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
hidden_states: torch.Tensor,
qr: torch.Tensor,
kv: torch.Tensor,
qr_scale: torch.Tensor | None,
kv_score: torch.Tensor,
indexer_kv_score: torch.Tensor,
indexer_weights: torch.Tensor,
@@ -446,7 +462,9 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
aux_streams = self.aux_stream_list
def project_query_and_cache_kv() -> torch.Tensor:
q = self.wq_b(qr).view(-1, self.n_local_heads, self.head_dim)
q = self._wq_b_proj(qr, qr_scale).view(
-1, self.n_local_heads, self.head_dim
)
return self._fused_qnorm_rope_kv_insert(q, kv, positions, attn_metadata)
index_q: torch.Tensor | None = None
@@ -468,6 +486,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
indexer_weights,
positions,
self.indexer_rotary_emb,
qr_scale,
),
lambda: compressor(kv_score, positions, self.rotary_emb),
],
@@ -507,6 +526,24 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
qr_kv, _ = self.fused_wqa_wkv(hidden_states)
return qr_kv
def _wq_b_proj(
self, qr: torch.Tensor, qr_scale: torch.Tensor | None = None
) -> torch.Tensor:
"""Project the q-lora input to the query heads.
``qr_scale`` is only set by the ROCm fused path, where ``qr`` is
already fp8-quantized with per-1x128 scales and must bypass the
linear's internal re-quantization (apply_weights would quantize
the fp8 input again).
"""
if qr_scale is None:
return self.wq_b(qr)
from vllm.models.deepseek_v4.amd.rocm import (
apply_pre_quantized_block_scaled_mm,
)
return apply_pre_quantized_block_scaled_mm(self.wq_b, qr, qr_scale)
def _run_parallel_input_projections(
self, hidden_states: torch.Tensor
) -> tuple[
@@ -881,6 +918,7 @@ class DeepseekV4Indexer(nn.Module):
indexer_weights: torch.Tensor,
positions: torch.Tensor,
rotary_emb: nn.Module,
qr_scale: torch.Tensor | None = None,
) -> tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]:
compressor = self.compressor
@@ -911,8 +949,7 @@ class DeepseekV4Indexer(nn.Module):
return None, None, None
def wq_b_and_q_quant():
# ReplicatedLinear returns (output, bias); bias is None.
q, _ = self.wq_b(qr)
q = self._wq_b_proj(qr, qr_scale)
q = q.view(-1, self.n_head, self.head_dim)
return fused_indexer_q_rope_quant(
positions,
@@ -938,3 +975,22 @@ class DeepseekV4Indexer(nn.Module):
else:
q, q_scale = q_quant, None
return q, q_scale, weights
def _wq_b_proj(
self, qr: torch.Tensor, qr_scale: torch.Tensor | None = None
) -> torch.Tensor:
"""Project the q-lora input to the indexer query heads.
Same bypass as ``DeepseekV4Attention._wq_b_proj``: on ROCm the fused
norm path hands over pre-quantized fp8 ``qr`` with per-1x128 scales,
so the linear's internal re-quantization must be skipped.
"""
if qr_scale is None:
# ReplicatedLinear returns (output, bias); bias is None.
q, _ = self.wq_b(qr)
return q
from vllm.models.deepseek_v4.amd.rocm import (
apply_pre_quantized_block_scaled_mm,
)
return apply_pre_quantized_block_scaled_mm(self.wq_b, qr, qr_scale)