[Feat][DSV4] Fuse q pad into deepseek v4 fused kernel (#43162)
This commit is contained in:
@@ -53,7 +53,7 @@ from vllm.model_executor.models.utils import (
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.models.deepseek_v4.nvidia.ops.attention import (
|
||||
from vllm.models.deepseek_v4.attention import (
|
||||
DeepseekV4Indexer,
|
||||
DeepseekV4MLAModules,
|
||||
DeepseekV4MultiHeadLatentAttentionWrapper,
|
||||
|
||||
@@ -32,7 +32,7 @@ from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.models.deepseek_v4.nvidia.ops.attention import (
|
||||
from vllm.models.deepseek_v4.attention import (
|
||||
DeepseekV4MLAAttention,
|
||||
)
|
||||
|
||||
@@ -592,6 +592,10 @@ class DeepseekV4ROCMAiterMLASparseImpl(DeepseekV4SparseMLAAttentionImpl):
|
||||
|
||||
backend_cls = DeepseekV4ROCMAiterMLASparseBackend
|
||||
|
||||
@classmethod
|
||||
def get_padded_num_q_heads(cls, num_heads: int) -> int:
|
||||
return num_heads
|
||||
|
||||
@classmethod
|
||||
def forward_mqa( # type: ignore[override]
|
||||
cls,
|
||||
|
||||
+23
-40
@@ -156,18 +156,6 @@ class DeepseekV4MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
self.head_dim = head_dim
|
||||
self.scale = scale
|
||||
|
||||
# FlashMLA sparse kernel only supports 64 or 128 heads; pad up to the
|
||||
# next supported size. Must match DeepseekV4MLAAttention.padded_heads.
|
||||
if num_heads <= 64:
|
||||
self.padded_heads = 64
|
||||
elif num_heads <= 128:
|
||||
self.padded_heads = 128
|
||||
else:
|
||||
raise ValueError(
|
||||
f"DeepseekV4 attention does not support {num_heads} heads "
|
||||
"(must be <= 128)."
|
||||
)
|
||||
|
||||
self.q_lora_rank = q_lora_rank
|
||||
self.kv_lora_rank = kv_lora_rank
|
||||
self.window_size = window_size
|
||||
@@ -263,6 +251,9 @@ class DeepseekV4MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
indexer=self.indexer,
|
||||
topk_indices_buffer=self.topk_indices_buffer,
|
||||
)
|
||||
# Mirror the inner layer's padded head count (single source of truth).
|
||||
self.padded_heads = self.mla_attn.padded_heads
|
||||
|
||||
# Register this layer in the compilation config's static forward context
|
||||
# This allows the custom op to retrieve the layer during execution
|
||||
compilation_config = mla_modules.vllm_config.compilation_config
|
||||
@@ -450,7 +441,7 @@ class DeepseekV4MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
|
||||
def wq_b_kv_insert() -> torch.Tensor:
|
||||
q = self.wq_b(qr).view(-1, self.n_local_heads, self.head_dim)
|
||||
self._fused_qnorm_rope_kv_insert(q, kv, positions, attn_metadata)
|
||||
q = self._fused_qnorm_rope_kv_insert(q, kv, positions, attn_metadata)
|
||||
return q
|
||||
|
||||
# 3-way overlap (matches TRT-LLM PR #14142 Level 1): default runs
|
||||
@@ -484,7 +475,7 @@ class DeepseekV4MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
|
||||
def wq_b_kv_insert() -> torch.Tensor:
|
||||
q = self.wq_b(qr).view(-1, self.n_local_heads, self.head_dim)
|
||||
self._fused_qnorm_rope_kv_insert(q, kv, positions, attn_metadata)
|
||||
q = self._fused_qnorm_rope_kv_insert(q, kv, positions, attn_metadata)
|
||||
return q
|
||||
|
||||
q, _ = maybe_execute_in_parallel(
|
||||
@@ -497,12 +488,7 @@ class DeepseekV4MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
else:
|
||||
# SWA-only layer: no compressor, no overlap.
|
||||
q = self.wq_b(qr).view(-1, self.n_local_heads, self.head_dim)
|
||||
self._fused_qnorm_rope_kv_insert(q, kv, positions, attn_metadata)
|
||||
|
||||
# Pad q to FlashMLA-required head count (64 or 128)
|
||||
if self.n_local_heads < self.padded_heads:
|
||||
pad_size = self.padded_heads - self.n_local_heads
|
||||
q = F.pad(q, (0, 0, 0, pad_size), value=0.0)
|
||||
q = self._fused_qnorm_rope_kv_insert(q, kv, positions, attn_metadata)
|
||||
|
||||
# MLA attention writes into the pre-allocated `out` buffer
|
||||
# ([num_tokens, padded_heads, head_dim]).
|
||||
@@ -516,9 +502,17 @@ class DeepseekV4MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
attn_metadata: (
|
||||
dict[str, AttentionMetadata] | list[dict[str, AttentionMetadata]] | None
|
||||
),
|
||||
) -> None:
|
||||
) -> torch.Tensor:
|
||||
if not isinstance(attn_metadata, dict):
|
||||
return
|
||||
# Profile run: kernel doesn't fire; produce a padded tensor so
|
||||
# downstream FlashMLA gets the right shape.
|
||||
if self.n_local_heads < self.padded_heads:
|
||||
return F.pad(
|
||||
q,
|
||||
(0, 0, 0, self.padded_heads - self.n_local_heads),
|
||||
value=0.0,
|
||||
)
|
||||
return q
|
||||
|
||||
swa_metadata = cast(
|
||||
"DeepseekSparseSWAMetadata | None",
|
||||
@@ -530,16 +524,19 @@ class DeepseekV4MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
swa_kv_cache_2d = swa_kv_cache.view(swa_kv_cache.shape[0], -1)
|
||||
|
||||
# Horizontally fused:
|
||||
# Q side: q_head_norm (per-head RMSNorm, no weight) + GPT-J RoPE
|
||||
# Q side: q_head_norm (per-head RMSNorm, no weight) + GPT-J RoPE,
|
||||
# with zero-fill for the padding head slots. The kernel
|
||||
# allocates and returns the padded q tensor.
|
||||
# KV side: GPT-J RoPE + UE8M0 FP8 quant + paged cache insert
|
||||
# kv is unchanged; mla_attn reads kv solely via swa_kv_cache.
|
||||
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
return torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
q,
|
||||
kv,
|
||||
swa_kv_cache_2d,
|
||||
swa_metadata.slot_mapping,
|
||||
positions.to(torch.int64),
|
||||
self.rotary_emb.cos_sin_cache,
|
||||
self.padded_heads,
|
||||
self.eps,
|
||||
swa_metadata.block_size,
|
||||
)
|
||||
@@ -607,9 +604,6 @@ direct_register_custom_op(
|
||||
|
||||
|
||||
class DeepseekV4MLAAttention(nn.Module, AttentionLayerBase):
|
||||
# FlashMLA FP8 sparse only supports 64 or 128 heads
|
||||
SUPPORTED_HEAD_COUNTS = (64, 128)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_heads: int,
|
||||
@@ -655,19 +649,8 @@ class DeepseekV4MLAAttention(nn.Module, AttentionLayerBase):
|
||||
self.aux_stream = aux_stream
|
||||
self.ln_events = [torch.cuda.Event(), torch.cuda.Event()]
|
||||
|
||||
# Determine padded head count for FlashMLA
|
||||
if num_heads not in self.SUPPORTED_HEAD_COUNTS:
|
||||
if num_heads < 64:
|
||||
self.padded_heads = 64
|
||||
elif num_heads < 128:
|
||||
self.padded_heads = 128
|
||||
else:
|
||||
raise ValueError(
|
||||
f"DeepseekV4MLAAttention does not support {num_heads} heads. "
|
||||
f"Supported: <= 128 (will be padded to 64 or 128)"
|
||||
)
|
||||
else:
|
||||
self.padded_heads = num_heads
|
||||
# Padded Q head count is dictated by the selected impl.
|
||||
self.padded_heads = self.impl_cls.get_padded_num_q_heads(num_heads)
|
||||
|
||||
# Store attention sink
|
||||
assert attn_sink is not None
|
||||
@@ -28,7 +28,7 @@ from vllm.v1.attention.ops.flashmla import (
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.models.deepseek_v4.nvidia.ops.attention import (
|
||||
from vllm.models.deepseek_v4.attention import (
|
||||
DeepseekV4MLAAttention,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
@@ -63,6 +63,18 @@ class DeepseekV4SparseMLAAttentionImpl(SparseMLAAttentionImpl[FlashMLASparseMeta
|
||||
) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def get_padded_num_q_heads(cls, num_heads: int) -> int:
|
||||
"""Q head count the backend wants q allocated at.
|
||||
|
||||
The MLA wrapper allocates the q/output buffers at
|
||||
``[N, get_padded_num_q_heads(n_local_heads), head_dim]``. Must
|
||||
satisfy ``result >= num_heads``. Backends with no padding constraint
|
||||
return ``num_heads``.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class DeepseekV4FlashMLASparseBackend(FlashMLASparseBackend):
|
||||
@staticmethod
|
||||
@@ -104,6 +116,16 @@ class DeepseekV4FlashMLASparseImpl(DeepseekV4SparseMLAAttentionImpl):
|
||||
|
||||
backend_cls = DeepseekV4FlashMLASparseBackend
|
||||
|
||||
@classmethod
|
||||
def get_padded_num_q_heads(cls, num_heads: int) -> int:
|
||||
# FP8 decode kernel only supports h_q = 64 or 128.
|
||||
if num_heads > 128:
|
||||
raise ValueError(
|
||||
f"DeepseekV4 FlashMLA does not support {num_heads} heads "
|
||||
"(FP8 decode kernel requires h_q in {64, 128})."
|
||||
)
|
||||
return 64 if num_heads <= 64 else 128
|
||||
|
||||
@classmethod
|
||||
def forward_mqa( # type: ignore[override]
|
||||
cls,
|
||||
|
||||
@@ -54,7 +54,7 @@ from vllm.model_executor.models.utils import (
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.models.deepseek_v4.nvidia.ops.attention import (
|
||||
from vllm.models.deepseek_v4.attention import (
|
||||
DeepseekV4Indexer,
|
||||
DeepseekV4MLAModules,
|
||||
DeepseekV4MultiHeadLatentAttentionWrapper,
|
||||
|
||||
Reference in New Issue
Block a user