[Attention Backend] add HPC-Ops Attention backend (#46020)
Signed-off-by: chengvjiang <chengvjiang@tencent.com> Co-authored-by: chengvjiang <chengvjiang@tencent.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
chengvjiang
Andreas Karatzas
parent
8e9d70fdd5
commit
8cf7c4d8ad
@@ -48,6 +48,7 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
GateLinear,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.hpc import HpcRopeNorm, QkNormPolicy
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -234,6 +235,7 @@ class HYV3Attention(nn.Module):
|
||||
dual_chunk_attention_config: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.dtype = torch.get_default_dtype()
|
||||
self.hidden_size = hidden_size
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
self.total_num_heads = num_heads
|
||||
@@ -276,11 +278,18 @@ class HYV3Attention(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
# When the HPC fused RoPE+QK-Norm path is enabled, the RoPE cos/sin
|
||||
# cache must be float32 to match the HPC kernel's expectations.
|
||||
kv_cache_dtype = cache_config.cache_dtype if cache_config else "auto"
|
||||
rope_support = HpcRopeNorm.support(
|
||||
self.num_heads, self.num_kv_heads, self.head_dim, kv_cache_dtype
|
||||
)
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
max_position=max_position_embeddings,
|
||||
rope_parameters=rope_parameters,
|
||||
is_neox_style=True,
|
||||
dtype=torch.float32 if rope_support else torch.get_default_dtype(),
|
||||
)
|
||||
self.attn = Attention(
|
||||
self.num_heads,
|
||||
@@ -295,6 +304,27 @@ class HYV3Attention(nn.Module):
|
||||
self.q_norm = RMSNorm(self.head_dim, rms_norm_eps)
|
||||
self.k_norm = RMSNorm(self.head_dim, rms_norm_eps)
|
||||
|
||||
# HPC fused RoPE + QK-Norm + KV-Cache-Write (+ optional FP8 Q quant).
|
||||
# HunYuan V3 applies QK-Norm *before* RoPE, so NORM_THEN_ROPE.
|
||||
self.hpc_rope_norm: HpcRopeNorm | None = None
|
||||
if rope_support:
|
||||
self.hpc_rope_norm = HpcRopeNorm(
|
||||
num_heads=self.num_heads,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_dim=self.head_dim,
|
||||
cos_sin_cache=self.rotary_emb.cos_sin_cache,
|
||||
use_qk_norm=self.use_qk_norm,
|
||||
fallback_qnorm=self.q_norm if self.use_qk_norm else None,
|
||||
fallback_knorm=self.k_norm if self.use_qk_norm else None,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
layer_name=self.attn.layer_name,
|
||||
qk_norm_policy=QkNormPolicy.NORM_THEN_ROPE,
|
||||
)
|
||||
# FP8 Q is produced by HpcRopeNorm, so the attention layer must not
|
||||
# re-quantize the query.
|
||||
if self.hpc_rope_norm.use_fp8 and hasattr(self.attn, "query_quant"):
|
||||
self.attn.query_quant = None
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
@@ -303,20 +333,28 @@ class HYV3Attention(nn.Module):
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
output_shape = None
|
||||
if self.use_qk_norm:
|
||||
q_by_head = q.view(
|
||||
*q.shape[:-1], q.shape[-1] // self.head_dim, self.head_dim
|
||||
)
|
||||
q_by_head = self.q_norm(q_by_head)
|
||||
q = q_by_head.view(q.shape)
|
||||
if self.hpc_rope_norm is not None:
|
||||
# HPC handles QK-Norm + RoPE + KV-cache write (+ optional FP8 Q
|
||||
# quant) internally and returns the processed query. K/V are
|
||||
# written into the paged cache by the fused op.
|
||||
q = self.hpc_rope_norm(qkv, self.attn.layer_name)
|
||||
q = q.view(-1, self.num_heads * self.head_dim)
|
||||
attn_output = self.attn(q, k, v, output_shape, self.dtype)
|
||||
else:
|
||||
if self.use_qk_norm:
|
||||
q_by_head = q.view(
|
||||
*q.shape[:-1], q.shape[-1] // self.head_dim, self.head_dim
|
||||
)
|
||||
q_by_head = self.q_norm(q_by_head)
|
||||
q = q_by_head.view(q.shape)
|
||||
|
||||
k_by_head = k.view(
|
||||
*k.shape[:-1], k.shape[-1] // self.head_dim, self.head_dim
|
||||
)
|
||||
k_by_head = self.k_norm(k_by_head)
|
||||
k = k_by_head.view(k.shape)
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
attn_output = self.attn(q, k, v, output_shape)
|
||||
k_by_head = k.view(
|
||||
*k.shape[:-1], k.shape[-1] // self.head_dim, self.head_dim
|
||||
)
|
||||
k_by_head = self.k_norm(k_by_head)
|
||||
k = k_by_head.view(k.shape)
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
attn_output = self.attn(q, k, v, output_shape)
|
||||
attn_output = attn_output.view(q.shape[0], -1)
|
||||
output, _ = self.o_proj(attn_output)
|
||||
return output
|
||||
|
||||
Reference in New Issue
Block a user