[ROCm][Perf] Fuse SWA q/kv RMSNorm and q FP8 group quant for DeepSeek-V4 (#53540)
Signed-off-by: Shanshan Shen <87969357+shen-shanshan@users.noreply.github.com>
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import functools
|
||||
from dataclasses import dataclass
|
||||
from typing import cast
|
||||
|
||||
@@ -60,6 +61,36 @@ def _build_indptr_from_lengths(lengths: torch.Tensor) -> torch.Tensor:
|
||||
return indptr
|
||||
|
||||
|
||||
def apply_pre_quantized_block_scaled_mm(
|
||||
linear: torch.nn.Module,
|
||||
x_fp8: torch.Tensor,
|
||||
x_scale: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Block-scaled fp8 GEMM on pre-quantized activations.
|
||||
|
||||
The fused q/kv norm kernel writes fp8 qr + per-1x128 scales; this
|
||||
drives the linear's block-scaled GEMM directly with them, bypassing
|
||||
apply_weights which would re-quantize the fp8 input. Only valid for
|
||||
the wq_b-style column/replicated linears: their output is the local
|
||||
TP shard, so no all-reduce is needed.
|
||||
"""
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.BlockScaledMMLinearKernel import (
|
||||
FP8BlockParams,
|
||||
)
|
||||
|
||||
params = FP8BlockParams.from_layer(linear)
|
||||
weight_scale = (
|
||||
params.weight_scale
|
||||
if params.weight_scale_inv is None
|
||||
else params.weight_scale_inv
|
||||
)
|
||||
kernel = linear.quant_method.fp8_linear
|
||||
out = kernel.apply_block_scaled_mm(
|
||||
A=x_fp8, B=params.weight, As=x_scale, Bs=weight_scale
|
||||
)
|
||||
return out.to(dtype=kernel.config.out_dtype)
|
||||
|
||||
|
||||
# ROCm sparse prefill keeps this dense combine local so AMD-specific SWA changes
|
||||
# do not touch the shared DeepSeek V4 cache utilities.
|
||||
_SPARSE_PREFILL_TOPK_ALIGNMENT = 128
|
||||
@@ -621,6 +652,71 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
|
||||
indexer_weights, _ = indexer.weights_proj(hidden_states)
|
||||
return qr_kv, kv_score, indexer_kv_score, indexer_weights
|
||||
|
||||
@functools.cached_property
|
||||
def _wq_b_uses_aiter_block_scaled(self) -> bool:
|
||||
"""True when both wq_b GEMMs run the aiter block-scaled fp8 kernel.
|
||||
|
||||
Cached: the linear kernels and the aiter env gates are fixed once
|
||||
the model is built, so this is evaluated at the first forward
|
||||
only.
|
||||
|
||||
The fused norm+quant path is only valid if the quant and GEMM it
|
||||
replaces are exactly the aiter ones; otherwise fall back to the
|
||||
shared path.
|
||||
"""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.kernels.linear.scaled_mm import (
|
||||
Fp8BlockScaledMMLinearKernel,
|
||||
)
|
||||
|
||||
if not rocm_aiter_ops.is_linear_fp8_enabled():
|
||||
return False
|
||||
|
||||
linears = [self.wq_b]
|
||||
if self.indexer is not None:
|
||||
linears.append(self.indexer.wq_b)
|
||||
for linear in linears:
|
||||
kernel = getattr(getattr(linear, "quant_method", None), "fp8_linear", None)
|
||||
if not isinstance(kernel, Fp8BlockScaledMMLinearKernel):
|
||||
return False
|
||||
return True
|
||||
|
||||
def _split_qkv_and_norm(
|
||||
self, qr_kv: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor]:
|
||||
"""Fuse q/kv RMSNorm + per-1x128 fp8 q quant into one aiter kernel.
|
||||
|
||||
The shared path norms q and kv in one triton kernel and the wq_b
|
||||
linears then re-read the bf16 qr to quantize it. The aiter kernel
|
||||
computes both RMSNorms (fp32 accumulate) and the fp8 group quant
|
||||
in a single pass, writing fp8 qr + group scales directly; both
|
||||
wq_b GEMMs (attention and indexer) then consume that pair and
|
||||
skip their own input quant. kv stays bf16: the fused insert
|
||||
kernel RoPE/quantizes it itself. Falls back to the shared path
|
||||
when the aiter linear path is not active.
|
||||
"""
|
||||
qr, kv = qr_kv.split([self.q_lora_rank, self.head_dim], dim=-1)
|
||||
if not (
|
||||
qr.dim() == 2
|
||||
and qr.shape[0] > 0
|
||||
and self.q_lora_rank % 128 == 0
|
||||
and self._wq_b_uses_aiter_block_scaled
|
||||
):
|
||||
return super()._split_qkv_and_norm(qr_kv)
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
return rocm_aiter_ops.fused_qk_rmsnorm_group_quant(
|
||||
q=qr,
|
||||
q_weight=self.q_norm.weight.data,
|
||||
q_epsilon=self.eps,
|
||||
kv=kv,
|
||||
kv_weight=self.kv_norm.weight.data,
|
||||
kv_epsilon=self.eps,
|
||||
group_size=128,
|
||||
transpose_scale=False,
|
||||
)
|
||||
|
||||
def _o_proj(self, o: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
|
||||
# ROCm BF16 reference wo_a path (inverse RoPE + einsum) + wo_b.
|
||||
z = rocm_inv_rope_einsum(
|
||||
|
||||
@@ -373,19 +373,13 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
qr_kv, kv_score, indexer_kv_score, indexer_weights = (
|
||||
self._run_parallel_input_projections(hidden_states)
|
||||
)
|
||||
qr, kv = qr_kv.split([self.q_lora_rank, self.head_dim], dim=-1)
|
||||
qr, kv = fused_q_kv_rmsnorm(
|
||||
qr,
|
||||
kv,
|
||||
self.q_norm.weight.data,
|
||||
self.kv_norm.weight.data,
|
||||
self.eps,
|
||||
)
|
||||
qr, qr_scale, kv = self._split_qkv_and_norm(qr_kv)
|
||||
|
||||
self._prepare_and_attn_fn(
|
||||
hidden_states,
|
||||
qr,
|
||||
kv,
|
||||
qr_scale,
|
||||
kv_score,
|
||||
indexer_kv_score,
|
||||
indexer_weights,
|
||||
@@ -397,12 +391,32 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
# Inverse-RoPE + wo_a + wo_b output projection (platform-specific).
|
||||
return self._o_proj(o, positions)
|
||||
|
||||
def _split_qkv_and_norm(
|
||||
self, qr_kv: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor]:
|
||||
"""Split the fused q-lora / kv projection and RMSNorm both halves.
|
||||
|
||||
``qr_scale`` is None on the shared path. The ROCm subclass returns
|
||||
a pre-quantized fp8 ``qr`` together with its per-1x128 fp32 scales,
|
||||
which the downstream wq_b projections consume directly.
|
||||
"""
|
||||
qr, kv = qr_kv.split([self.q_lora_rank, self.head_dim], dim=-1)
|
||||
qr, kv = fused_q_kv_rmsnorm(
|
||||
qr,
|
||||
kv,
|
||||
self.q_norm.weight.data,
|
||||
self.kv_norm.weight.data,
|
||||
self.eps,
|
||||
)
|
||||
return qr, None, kv
|
||||
|
||||
@eager_break_during_capture
|
||||
def _prepare_and_attn_eager(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
qr: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
qr_scale: torch.Tensor | None,
|
||||
kv_score: torch.Tensor,
|
||||
indexer_kv_score: torch.Tensor,
|
||||
indexer_weights: torch.Tensor,
|
||||
@@ -418,6 +432,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
hidden_states,
|
||||
qr,
|
||||
kv,
|
||||
qr_scale,
|
||||
kv_score,
|
||||
indexer_kv_score,
|
||||
indexer_weights,
|
||||
@@ -430,6 +445,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
hidden_states: torch.Tensor,
|
||||
qr: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
qr_scale: torch.Tensor | None,
|
||||
kv_score: torch.Tensor,
|
||||
indexer_kv_score: torch.Tensor,
|
||||
indexer_weights: torch.Tensor,
|
||||
@@ -446,7 +462,9 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
aux_streams = self.aux_stream_list
|
||||
|
||||
def project_query_and_cache_kv() -> torch.Tensor:
|
||||
q = self.wq_b(qr).view(-1, self.n_local_heads, self.head_dim)
|
||||
q = self._wq_b_proj(qr, qr_scale).view(
|
||||
-1, self.n_local_heads, self.head_dim
|
||||
)
|
||||
return self._fused_qnorm_rope_kv_insert(q, kv, positions, attn_metadata)
|
||||
|
||||
index_q: torch.Tensor | None = None
|
||||
@@ -468,6 +486,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
indexer_weights,
|
||||
positions,
|
||||
self.indexer_rotary_emb,
|
||||
qr_scale,
|
||||
),
|
||||
lambda: compressor(kv_score, positions, self.rotary_emb),
|
||||
],
|
||||
@@ -507,6 +526,24 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
qr_kv, _ = self.fused_wqa_wkv(hidden_states)
|
||||
return qr_kv
|
||||
|
||||
def _wq_b_proj(
|
||||
self, qr: torch.Tensor, qr_scale: torch.Tensor | None = None
|
||||
) -> torch.Tensor:
|
||||
"""Project the q-lora input to the query heads.
|
||||
|
||||
``qr_scale`` is only set by the ROCm fused path, where ``qr`` is
|
||||
already fp8-quantized with per-1x128 scales and must bypass the
|
||||
linear's internal re-quantization (apply_weights would quantize
|
||||
the fp8 input again).
|
||||
"""
|
||||
if qr_scale is None:
|
||||
return self.wq_b(qr)
|
||||
from vllm.models.deepseek_v4.amd.rocm import (
|
||||
apply_pre_quantized_block_scaled_mm,
|
||||
)
|
||||
|
||||
return apply_pre_quantized_block_scaled_mm(self.wq_b, qr, qr_scale)
|
||||
|
||||
def _run_parallel_input_projections(
|
||||
self, hidden_states: torch.Tensor
|
||||
) -> tuple[
|
||||
@@ -881,6 +918,7 @@ class DeepseekV4Indexer(nn.Module):
|
||||
indexer_weights: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
rotary_emb: nn.Module,
|
||||
qr_scale: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]:
|
||||
compressor = self.compressor
|
||||
|
||||
@@ -911,8 +949,7 @@ class DeepseekV4Indexer(nn.Module):
|
||||
return None, None, None
|
||||
|
||||
def wq_b_and_q_quant():
|
||||
# ReplicatedLinear returns (output, bias); bias is None.
|
||||
q, _ = self.wq_b(qr)
|
||||
q = self._wq_b_proj(qr, qr_scale)
|
||||
q = q.view(-1, self.n_head, self.head_dim)
|
||||
return fused_indexer_q_rope_quant(
|
||||
positions,
|
||||
@@ -938,3 +975,22 @@ class DeepseekV4Indexer(nn.Module):
|
||||
else:
|
||||
q, q_scale = q_quant, None
|
||||
return q, q_scale, weights
|
||||
|
||||
def _wq_b_proj(
|
||||
self, qr: torch.Tensor, qr_scale: torch.Tensor | None = None
|
||||
) -> torch.Tensor:
|
||||
"""Project the q-lora input to the indexer query heads.
|
||||
|
||||
Same bypass as ``DeepseekV4Attention._wq_b_proj``: on ROCm the fused
|
||||
norm path hands over pre-quantized fp8 ``qr`` with per-1x128 scales,
|
||||
so the linear's internal re-quantization must be skipped.
|
||||
"""
|
||||
if qr_scale is None:
|
||||
# ReplicatedLinear returns (output, bias); bias is None.
|
||||
q, _ = self.wq_b(qr)
|
||||
return q
|
||||
from vllm.models.deepseek_v4.amd.rocm import (
|
||||
apply_pre_quantized_block_scaled_mm,
|
||||
)
|
||||
|
||||
return apply_pre_quantized_block_scaled_mm(self.wq_b, qr, qr_scale)
|
||||
|
||||
Reference in New Issue
Block a user