315 lines
11 KiB
Python
315 lines
11 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
from typing import cast
|
|
|
|
import torch
|
|
|
|
from vllm._aiter_ops import rocm_aiter_ops
|
|
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
|
from vllm.forward_context import get_forward_context
|
|
from vllm.model_executor.layers.attention.mla_attention import MLACommonMetadata
|
|
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
|
|
from vllm.model_executor.models.deepseek_v2 import DeepseekV32IndexerCache
|
|
from vllm.models.deepseek_v32.attention import DeepseekV32Attention, DeepseekV32Indexer
|
|
from vllm.models.deepseek_v32.common.kernels import fused_norm_rope, fused_q
|
|
from vllm.platforms import current_platform
|
|
from vllm.utils.torch_utils import is_quantized_kv_cache
|
|
from vllm.v1.attention.backends.mla.indexer import DeepseekV32IndexerBackend
|
|
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
|
|
ROCMAiterMLASparseBackend,
|
|
)
|
|
|
|
|
|
class DeepseekV32ROCmIndexerCache(DeepseekV32IndexerCache):
|
|
def get_attn_backend(self):
|
|
return DeepseekV32IndexerBackend
|
|
|
|
@property
|
|
def uses_shuffled_layout(self) -> bool:
|
|
# aiter's gather/insert pair shuffles the cache above block size 1:
|
|
# [n_blocks, blk/16, head_dim/16, 16, 16] instead of [n_blocks, blk, head_dim].
|
|
return self.kv_cache.ndim == 3 and self.kv_cache.shape[1] != 1
|
|
|
|
|
|
class DeepseekV32ROCmIndexer(DeepseekV32Indexer):
|
|
indexer_cache_cls = DeepseekV32ROCmIndexerCache
|
|
|
|
|
|
class DeepseekV32MLAAttention(DeepseekV32Attention):
|
|
indexer_cls = DeepseekV32ROCmIndexer
|
|
|
|
def __init__(self, vllm_config, config, prefix, topk_indices_buffer=None):
|
|
super().__init__(
|
|
vllm_config,
|
|
config,
|
|
prefix,
|
|
topk_indices_buffer,
|
|
attn_backend=ROCMAiterMLASparseBackend,
|
|
)
|
|
|
|
self.indexer_op: SparseAttnIndexer | None = None
|
|
if self.indexer is not None:
|
|
self.indexer_op = SparseAttnIndexer(
|
|
self.indexer.k_cache,
|
|
self.indexer.quant_block_size,
|
|
self.indexer.scale_fmt,
|
|
self.indexer.topk_tokens,
|
|
self.indexer.head_dim,
|
|
self.indexer.max_model_len,
|
|
self.indexer.max_total_seq_len,
|
|
topk_indices_buffer,
|
|
skip_k_cache_insert=True,
|
|
)
|
|
self._fp8_kv = is_quantized_kv_cache(self.kv_cache_dtype)
|
|
self._fp8_kv_needs_view = self._fp8_kv and self.kv_cache_dtype != "fp8_ds_mla"
|
|
|
|
def forward( # type: ignore[override]
|
|
self,
|
|
positions: torch.Tensor,
|
|
hidden_states: torch.Tensor,
|
|
) -> torch.Tensor:
|
|
qkv_lora = self.fused_qkv_a_proj(hidden_states)[0]
|
|
q_c, kv_c, k_pe = qkv_lora.split(
|
|
[self.q_lora_rank, self.kv_lora_rank, self.qk_rope_head_dim], dim=-1
|
|
)
|
|
|
|
if self.indexer is not None and not self.skip_topk:
|
|
kw = self.indexer.wk_weights_proj(hidden_states)[0]
|
|
index_k = kw[:, : self.indexer.head_dim]
|
|
index_weights = kw[:, self.indexer.head_dim :]
|
|
else:
|
|
index_k = None
|
|
index_weights = None
|
|
|
|
num_tokens = hidden_states.shape[0]
|
|
output = torch.empty(
|
|
(num_tokens, self.num_local_heads * self.v_head_dim),
|
|
dtype=hidden_states.dtype,
|
|
device=hidden_states.device,
|
|
)
|
|
self._fused_attention(
|
|
positions, q_c, kv_c, k_pe, index_k, index_weights, output
|
|
)
|
|
return self.o_proj(output)[0]
|
|
|
|
def _compute_ql_nope(self, q_c: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
|
q = self.q_b_proj(q_c)[0].view(-1, self.num_local_heads, self.qk_head_dim)
|
|
q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1)
|
|
q_nope = q_nope.transpose(0, 1) # (N, tokens, P)
|
|
|
|
if self.is_aiter_triton_fp4_bmm_enabled:
|
|
ql_nope = rocm_aiter_ops.batched_gemm_a16wfp4(
|
|
q_nope, self.W_K, self.W_K_scale, transpose_bm=True, prequant=True
|
|
)
|
|
elif self.is_aiter_triton_fp8_bmm_enabled:
|
|
ql_nope = rocm_aiter_ops.triton_fp8_bmm(
|
|
q_nope, self.W_K, self.W_K_scale, group_size=128, transpose_bm=True
|
|
)
|
|
else:
|
|
ql_nope = torch.bmm(q_nope, self.W_UK_T).transpose(0, 1)
|
|
|
|
return ql_nope, q_pe
|
|
|
|
def _run_indexer(
|
|
self,
|
|
q_c: torch.Tensor,
|
|
index_q_fp8: torch.Tensor | None,
|
|
index_weights_out: torch.Tensor | None,
|
|
) -> None:
|
|
"""Run the ROCm sparse indexer (forward_hip) if this layer has an indexer."""
|
|
if self.indexer_op is not None:
|
|
self.indexer_op.forward_hip(q_c, index_q_fp8, None, index_weights_out)
|
|
|
|
def _build_q_for_attn(
|
|
self,
|
|
ql_nope: torch.Tensor,
|
|
mqa_q: torch.Tensor,
|
|
num_actual: int,
|
|
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
|
if self._fp8_kv:
|
|
# fp8 KV: mqa_q is the full [ql_nope; q_pe] packed as fp8.
|
|
return mqa_q[:num_actual]
|
|
|
|
return (ql_nope[:num_actual], mqa_q[:num_actual])
|
|
|
|
def _compute_uv_out(
|
|
self,
|
|
attn_out: torch.Tensor,
|
|
output: torch.Tensor,
|
|
num_actual: int,
|
|
) -> None:
|
|
x = attn_out.view(
|
|
num_actual, self.num_local_heads, self.kv_lora_rank
|
|
).transpose(0, 1) # (N, tokens, L)
|
|
out_view = output[:num_actual].view(
|
|
num_actual, self.num_local_heads, self.v_head_dim
|
|
)
|
|
|
|
if self.is_aiter_triton_fp4_bmm_enabled:
|
|
rocm_aiter_ops.batched_gemm_a16wfp4(
|
|
x, self.W_V, self.W_V_scale, out_view, transpose_bm=True, prequant=True
|
|
)
|
|
elif self.is_aiter_triton_fp8_bmm_enabled:
|
|
rocm_aiter_ops.triton_fp8_bmm(
|
|
x,
|
|
self.W_V,
|
|
self.W_V_scale,
|
|
group_size=128,
|
|
transpose_bm=True,
|
|
YQ=out_view,
|
|
)
|
|
else:
|
|
torch.bmm(x, self.W_UV, out=out_view.transpose(0, 1))
|
|
|
|
@eager_break_during_capture
|
|
def _fused_attention(
|
|
self,
|
|
positions: torch.Tensor,
|
|
q_c: torch.Tensor,
|
|
kv_c: torch.Tensor,
|
|
k_pe: torch.Tensor,
|
|
index_k: torch.Tensor | None,
|
|
index_weights: torch.Tensor | None,
|
|
output: torch.Tensor,
|
|
) -> None:
|
|
forward_context = get_forward_context()
|
|
attn_metadata_raw = forward_context.attn_metadata
|
|
if isinstance(attn_metadata_raw, dict):
|
|
attn_metadata = attn_metadata_raw.get(self.layer_name)
|
|
elif isinstance(attn_metadata_raw, list):
|
|
attn_metadata = attn_metadata_raw[0].get(self.layer_name)
|
|
else:
|
|
attn_metadata = attn_metadata_raw
|
|
|
|
attn_metadata = cast(MLACommonMetadata | None, attn_metadata)
|
|
|
|
slot_mapping = forward_context.slot_mapping
|
|
assert isinstance(slot_mapping, dict)
|
|
mla_slot = slot_mapping.get(self.layer_name)
|
|
|
|
if self.indexer is not None and not self.skip_topk:
|
|
has_indexer = True
|
|
indexer_k_norm_w = self.indexer.k_norm.weight
|
|
indexer_k_norm_bias = self.indexer.k_norm.bias
|
|
indexer_k_norm_eps = self.indexer.k_norm.eps
|
|
indexer_k_rope_cos_sin_cache = self.indexer_rope_emb.cos_sin_cache
|
|
indexer_k_cache = self.indexer.k_cache.kv_cache
|
|
indexer_cache_shuffled = self.indexer.k_cache.uses_shuffled_layout
|
|
indexer_softmax_scale = self.indexer.softmax_scale
|
|
indexer_n_head_scale = self.indexer.n_head**-0.5
|
|
else:
|
|
has_indexer = False
|
|
indexer_k_norm_w = None
|
|
indexer_k_norm_bias = None
|
|
indexer_k_norm_eps = 1e-6
|
|
indexer_k_rope_cos_sin_cache = None
|
|
indexer_k_cache = None
|
|
indexer_cache_shuffled = False
|
|
indexer_softmax_scale = 0.0
|
|
indexer_n_head_scale = 0.0
|
|
|
|
if attn_metadata is None:
|
|
mla_kv_cache = None
|
|
mla_k_scale = None
|
|
indexer_k_cache = None
|
|
mla_slot = None
|
|
else:
|
|
mla_kv_cache = self.kv_cache
|
|
mla_k_scale = self._k_scale
|
|
|
|
use_mha = attn_metadata is not None and self._use_sparse_mha(attn_metadata)
|
|
kv_c_out = torch.empty_like(kv_c) if use_mha else None
|
|
k_pe_out = torch.empty_like(k_pe) if use_mha else None
|
|
|
|
q_c = fused_norm_rope(
|
|
positions,
|
|
q_c,
|
|
self.q_a_layernorm.weight,
|
|
self.q_a_layernorm.variance_epsilon,
|
|
kv_c,
|
|
self.kv_a_layernorm.weight,
|
|
self.kv_a_layernorm.variance_epsilon,
|
|
k_pe,
|
|
self.rotary_emb.cos_sin_cache,
|
|
index_k,
|
|
indexer_k_norm_w,
|
|
indexer_k_norm_bias,
|
|
indexer_k_norm_eps,
|
|
indexer_k_rope_cos_sin_cache,
|
|
self.topk_indices_buffer,
|
|
slot_mapping=mla_slot,
|
|
indexer_k_cache=indexer_k_cache,
|
|
indexer_cache_shuffled=indexer_cache_shuffled,
|
|
mla_kv_cache=mla_kv_cache,
|
|
mla_kv_cache_dtype=self.kv_cache_dtype,
|
|
mla_k_scale=mla_k_scale,
|
|
has_indexer=has_indexer,
|
|
index_rope_interleave=self._index_rope_interleave,
|
|
kv_c_out=kv_c_out,
|
|
k_pe_out=k_pe_out,
|
|
)
|
|
|
|
if use_mha:
|
|
q = self.q_b_proj(q_c)[0].view(-1, self.num_local_heads, self.qk_head_dim)
|
|
q_nope, q_pe = q.split(
|
|
[self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1
|
|
)
|
|
ql_nope = q_nope
|
|
else:
|
|
ql_nope, q_pe = self._compute_ql_nope(q_c)
|
|
|
|
if self.indexer is not None and not self.skip_topk:
|
|
index_q = self.indexer.wq_b(q_c)[0]
|
|
index_q = index_q.view(-1, self.indexer.n_head, self.indexer.head_dim)
|
|
else:
|
|
index_q = None
|
|
|
|
index_q_fp8, index_weights_out, mqa_q = fused_q(
|
|
positions,
|
|
q_pe,
|
|
self.rotary_emb.cos_sin_cache,
|
|
index_q,
|
|
self.indexer_rope_emb.cos_sin_cache if has_indexer else None,
|
|
ql_nope,
|
|
self._q_scale,
|
|
index_weights,
|
|
indexer_softmax_scale,
|
|
indexer_n_head_scale,
|
|
has_indexer=has_indexer,
|
|
index_rope_interleave=self._index_rope_interleave,
|
|
quantize_mqa=self._fp8_kv and not use_mha,
|
|
)
|
|
|
|
if self.indexer is not None and not self.skip_topk:
|
|
self._run_indexer(q_c, index_q_fp8, index_weights_out)
|
|
|
|
if attn_metadata is None:
|
|
output.zero_()
|
|
return
|
|
|
|
if use_mha:
|
|
assert kv_c_out is not None and k_pe_out is not None
|
|
mha_q = torch.cat((q_nope, mqa_q), dim=-1)
|
|
self.forward_impl(
|
|
mha_q,
|
|
kv_c_out,
|
|
k_pe_out.unsqueeze(1),
|
|
self.kv_cache,
|
|
attn_metadata,
|
|
output,
|
|
)
|
|
return
|
|
|
|
num_actual = attn_metadata.num_actual_tokens # type: ignore[attr-defined]
|
|
kv_cache = self.kv_cache
|
|
if self._fp8_kv_needs_view:
|
|
kv_cache = kv_cache.view(current_platform.fp8_dtype())
|
|
|
|
q_for_attn = self._build_q_for_attn(ql_nope, mqa_q, num_actual)
|
|
attn_out, _ = self.impl.forward_mqa( # type: ignore[attr-defined]
|
|
q_for_attn, kv_cache, attn_metadata, self
|
|
)
|
|
|
|
self._compute_uv_out(attn_out, output, num_actual)
|