[ROCm][Perf] Fuse SWA q/kv RMSNorm and q FP8 group quant for DeepSeek-V4 (#53540)

Signed-off-by: Shanshan Shen <87969357+shen-shanshan@users.noreply.github.com>
This commit is contained in:
Shanshan Shen
2026-08-27 09:37:54 -07:00
committed by GitHub
parent 9650dc73e2
commit 32ad1400d7
4 changed files with 333 additions and 12 deletions
+96
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools
from dataclasses import dataclass
from typing import cast
@@ -60,6 +61,36 @@ def _build_indptr_from_lengths(lengths: torch.Tensor) -> torch.Tensor:
return indptr
def apply_pre_quantized_block_scaled_mm(
linear: torch.nn.Module,
x_fp8: torch.Tensor,
x_scale: torch.Tensor,
) -> torch.Tensor:
"""Block-scaled fp8 GEMM on pre-quantized activations.
The fused q/kv norm kernel writes fp8 qr + per-1x128 scales; this
drives the linear's block-scaled GEMM directly with them, bypassing
apply_weights which would re-quantize the fp8 input. Only valid for
the wq_b-style column/replicated linears: their output is the local
TP shard, so no all-reduce is needed.
"""
from vllm.model_executor.kernels.linear.scaled_mm.BlockScaledMMLinearKernel import (
FP8BlockParams,
)
params = FP8BlockParams.from_layer(linear)
weight_scale = (
params.weight_scale
if params.weight_scale_inv is None
else params.weight_scale_inv
)
kernel = linear.quant_method.fp8_linear
out = kernel.apply_block_scaled_mm(
A=x_fp8, B=params.weight, As=x_scale, Bs=weight_scale
)
return out.to(dtype=kernel.config.out_dtype)
# ROCm sparse prefill keeps this dense combine local so AMD-specific SWA changes
# do not touch the shared DeepSeek V4 cache utilities.
_SPARSE_PREFILL_TOPK_ALIGNMENT = 128
@@ -621,6 +652,71 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
indexer_weights, _ = indexer.weights_proj(hidden_states)
return qr_kv, kv_score, indexer_kv_score, indexer_weights
@functools.cached_property
def _wq_b_uses_aiter_block_scaled(self) -> bool:
"""True when both wq_b GEMMs run the aiter block-scaled fp8 kernel.
Cached: the linear kernels and the aiter env gates are fixed once
the model is built, so this is evaluated at the first forward
only.
The fused norm+quant path is only valid if the quant and GEMM it
replaces are exactly the aiter ones; otherwise fall back to the
shared path.
"""
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.kernels.linear.scaled_mm import (
Fp8BlockScaledMMLinearKernel,
)
if not rocm_aiter_ops.is_linear_fp8_enabled():
return False
linears = [self.wq_b]
if self.indexer is not None:
linears.append(self.indexer.wq_b)
for linear in linears:
kernel = getattr(getattr(linear, "quant_method", None), "fp8_linear", None)
if not isinstance(kernel, Fp8BlockScaledMMLinearKernel):
return False
return True
def _split_qkv_and_norm(
self, qr_kv: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor]:
"""Fuse q/kv RMSNorm + per-1x128 fp8 q quant into one aiter kernel.
The shared path norms q and kv in one triton kernel and the wq_b
linears then re-read the bf16 qr to quantize it. The aiter kernel
computes both RMSNorms (fp32 accumulate) and the fp8 group quant
in a single pass, writing fp8 qr + group scales directly; both
wq_b GEMMs (attention and indexer) then consume that pair and
skip their own input quant. kv stays bf16: the fused insert
kernel RoPE/quantizes it itself. Falls back to the shared path
when the aiter linear path is not active.
"""
qr, kv = qr_kv.split([self.q_lora_rank, self.head_dim], dim=-1)
if not (
qr.dim() == 2
and qr.shape[0] > 0
and self.q_lora_rank % 128 == 0
and self._wq_b_uses_aiter_block_scaled
):
return super()._split_qkv_and_norm(qr_kv)
from vllm._aiter_ops import rocm_aiter_ops
return rocm_aiter_ops.fused_qk_rmsnorm_group_quant(
q=qr,
q_weight=self.q_norm.weight.data,
q_epsilon=self.eps,
kv=kv,
kv_weight=self.kv_norm.weight.data,
kv_epsilon=self.eps,
group_size=128,
transpose_scale=False,
)
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
# ROCm BF16 reference wo_a path (inverse RoPE + einsum) + wo_b.
z = rocm_inv_rope_einsum(
+67 -11
View File
@@ -373,19 +373,13 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
qr_kv, kv_score, indexer_kv_score, indexer_weights = (
self._run_parallel_input_projections(hidden_states)
)
qr, kv = qr_kv.split([self.q_lora_rank, self.head_dim], dim=-1)
qr, kv = fused_q_kv_rmsnorm(
qr,
kv,
self.q_norm.weight.data,
self.kv_norm.weight.data,
self.eps,
)
qr, qr_scale, kv = self._split_qkv_and_norm(qr_kv)
self._prepare_and_attn_fn(
hidden_states,
qr,
kv,
qr_scale,
kv_score,
indexer_kv_score,
indexer_weights,
@@ -397,12 +391,32 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
# Inverse-RoPE + wo_a + wo_b output projection (platform-specific).
return self._o_proj(o, positions)
def _split_qkv_and_norm(
self, qr_kv: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor]:
"""Split the fused q-lora / kv projection and RMSNorm both halves.
``qr_scale`` is None on the shared path. The ROCm subclass returns
a pre-quantized fp8 ``qr`` together with its per-1x128 fp32 scales,
which the downstream wq_b projections consume directly.
"""
qr, kv = qr_kv.split([self.q_lora_rank, self.head_dim], dim=-1)
qr, kv = fused_q_kv_rmsnorm(
qr,
kv,
self.q_norm.weight.data,
self.kv_norm.weight.data,
self.eps,
)
return qr, None, kv
@eager_break_during_capture
def _prepare_and_attn_eager(
self,
hidden_states: torch.Tensor,
qr: torch.Tensor,
kv: torch.Tensor,
qr_scale: torch.Tensor | None,
kv_score: torch.Tensor,
indexer_kv_score: torch.Tensor,
indexer_weights: torch.Tensor,
@@ -418,6 +432,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
hidden_states,
qr,
kv,
qr_scale,
kv_score,
indexer_kv_score,
indexer_weights,
@@ -430,6 +445,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
hidden_states: torch.Tensor,
qr: torch.Tensor,
kv: torch.Tensor,
qr_scale: torch.Tensor | None,
kv_score: torch.Tensor,
indexer_kv_score: torch.Tensor,
indexer_weights: torch.Tensor,
@@ -446,7 +462,9 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
aux_streams = self.aux_stream_list
def project_query_and_cache_kv() -> torch.Tensor:
q = self.wq_b(qr).view(-1, self.n_local_heads, self.head_dim)
q = self._wq_b_proj(qr, qr_scale).view(
-1, self.n_local_heads, self.head_dim
)
return self._fused_qnorm_rope_kv_insert(q, kv, positions, attn_metadata)
index_q: torch.Tensor | None = None
@@ -468,6 +486,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
indexer_weights,
positions,
self.indexer_rotary_emb,
qr_scale,
),
lambda: compressor(kv_score, positions, self.rotary_emb),
],
@@ -507,6 +526,24 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
qr_kv, _ = self.fused_wqa_wkv(hidden_states)
return qr_kv
def _wq_b_proj(
self, qr: torch.Tensor, qr_scale: torch.Tensor | None = None
) -> torch.Tensor:
"""Project the q-lora input to the query heads.
``qr_scale`` is only set by the ROCm fused path, where ``qr`` is
already fp8-quantized with per-1x128 scales and must bypass the
linear's internal re-quantization (apply_weights would quantize
the fp8 input again).
"""
if qr_scale is None:
return self.wq_b(qr)
from vllm.models.deepseek_v4.amd.rocm import (
apply_pre_quantized_block_scaled_mm,
)
return apply_pre_quantized_block_scaled_mm(self.wq_b, qr, qr_scale)
def _run_parallel_input_projections(
self, hidden_states: torch.Tensor
) -> tuple[
@@ -881,6 +918,7 @@ class DeepseekV4Indexer(nn.Module):
indexer_weights: torch.Tensor,
positions: torch.Tensor,
rotary_emb: nn.Module,
qr_scale: torch.Tensor | None = None,
) -> tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]:
compressor = self.compressor
@@ -911,8 +949,7 @@ class DeepseekV4Indexer(nn.Module):
return None, None, None
def wq_b_and_q_quant():
# ReplicatedLinear returns (output, bias); bias is None.
q, _ = self.wq_b(qr)
q = self._wq_b_proj(qr, qr_scale)
q = q.view(-1, self.n_head, self.head_dim)
return fused_indexer_q_rope_quant(
positions,
@@ -938,3 +975,22 @@ class DeepseekV4Indexer(nn.Module):
else:
q, q_scale = q_quant, None
return q, q_scale, weights
def _wq_b_proj(
self, qr: torch.Tensor, qr_scale: torch.Tensor | None = None
) -> torch.Tensor:
"""Project the q-lora input to the indexer query heads.
Same bypass as ``DeepseekV4Attention._wq_b_proj``: on ROCm the fused
norm path hands over pre-quantized fp8 ``qr`` with per-1x128 scales,
so the linear's internal re-quantization must be skipped.
"""
if qr_scale is None:
# ReplicatedLinear returns (output, bias); bias is None.
q, _ = self.wq_b(qr)
return q
from vllm.models.deepseek_v4.amd.rocm import (
apply_pre_quantized_block_scaled_mm,
)
return apply_pre_quantized_block_scaled_mm(self.wq_b, qr, qr_scale)