[Attention Backend] add HPC-Ops Attention backend (#46020)

Signed-off-by: chengvjiang <chengvjiang@tencent.com>
Co-authored-by: chengvjiang <chengvjiang@tencent.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Cheng Jiang
2026-06-30 18:17:43 +08:00
committed by GitHub
co-authored by chengvjiang Andreas Karatzas
parent 8e9d70fdd5
commit 8cf7c4d8ad
10 changed files with 978 additions and 14 deletions
+51 -13
View File
@@ -48,6 +48,7 @@ from vllm.model_executor.layers.fused_moe import (
GateLinear,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.hpc import HpcRopeNorm, QkNormPolicy
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -234,6 +235,7 @@ class HYV3Attention(nn.Module):
dual_chunk_attention_config: dict[str, Any] | None = None,
) -> None:
super().__init__()
self.dtype = torch.get_default_dtype()
self.hidden_size = hidden_size
tp_size = get_tensor_model_parallel_world_size()
self.total_num_heads = num_heads
@@ -276,11 +278,18 @@ class HYV3Attention(nn.Module):
quant_config=quant_config,
prefix=f"{prefix}.o_proj",
)
# When the HPC fused RoPE+QK-Norm path is enabled, the RoPE cos/sin
# cache must be float32 to match the HPC kernel's expectations.
kv_cache_dtype = cache_config.cache_dtype if cache_config else "auto"
rope_support = HpcRopeNorm.support(
self.num_heads, self.num_kv_heads, self.head_dim, kv_cache_dtype
)
self.rotary_emb = get_rope(
self.head_dim,
max_position=max_position_embeddings,
rope_parameters=rope_parameters,
is_neox_style=True,
dtype=torch.float32 if rope_support else torch.get_default_dtype(),
)
self.attn = Attention(
self.num_heads,
@@ -295,6 +304,27 @@ class HYV3Attention(nn.Module):
self.q_norm = RMSNorm(self.head_dim, rms_norm_eps)
self.k_norm = RMSNorm(self.head_dim, rms_norm_eps)
# HPC fused RoPE + QK-Norm + KV-Cache-Write (+ optional FP8 Q quant).
# HunYuan V3 applies QK-Norm *before* RoPE, so NORM_THEN_ROPE.
self.hpc_rope_norm: HpcRopeNorm | None = None
if rope_support:
self.hpc_rope_norm = HpcRopeNorm(
num_heads=self.num_heads,
num_kv_heads=self.num_kv_heads,
head_dim=self.head_dim,
cos_sin_cache=self.rotary_emb.cos_sin_cache,
use_qk_norm=self.use_qk_norm,
fallback_qnorm=self.q_norm if self.use_qk_norm else None,
fallback_knorm=self.k_norm if self.use_qk_norm else None,
kv_cache_dtype=kv_cache_dtype,
layer_name=self.attn.layer_name,
qk_norm_policy=QkNormPolicy.NORM_THEN_ROPE,
)
# FP8 Q is produced by HpcRopeNorm, so the attention layer must not
# re-quantize the query.
if self.hpc_rope_norm.use_fp8 and hasattr(self.attn, "query_quant"):
self.attn.query_quant = None
def forward(
self,
positions: torch.Tensor,
@@ -303,20 +333,28 @@ class HYV3Attention(nn.Module):
qkv, _ = self.qkv_proj(hidden_states)
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
output_shape = None
if self.use_qk_norm:
q_by_head = q.view(
*q.shape[:-1], q.shape[-1] // self.head_dim, self.head_dim
)
q_by_head = self.q_norm(q_by_head)
q = q_by_head.view(q.shape)
if self.hpc_rope_norm is not None:
# HPC handles QK-Norm + RoPE + KV-cache write (+ optional FP8 Q
# quant) internally and returns the processed query. K/V are
# written into the paged cache by the fused op.
q = self.hpc_rope_norm(qkv, self.attn.layer_name)
q = q.view(-1, self.num_heads * self.head_dim)
attn_output = self.attn(q, k, v, output_shape, self.dtype)
else:
if self.use_qk_norm:
q_by_head = q.view(
*q.shape[:-1], q.shape[-1] // self.head_dim, self.head_dim
)
q_by_head = self.q_norm(q_by_head)
q = q_by_head.view(q.shape)
k_by_head = k.view(
*k.shape[:-1], k.shape[-1] // self.head_dim, self.head_dim
)
k_by_head = self.k_norm(k_by_head)
k = k_by_head.view(k.shape)
q, k = self.rotary_emb(positions, q, k)
attn_output = self.attn(q, k, v, output_shape)
k_by_head = k.view(
*k.shape[:-1], k.shape[-1] // self.head_dim, self.head_dim
)
k_by_head = self.k_norm(k_by_head)
k = k_by_head.view(k.shape)
q, k = self.rotary_emb(positions, q, k)
attn_output = self.attn(q, k, v, output_shape)
attn_output = attn_output.view(q.shape[0], -1)
output, _ = self.o_proj(attn_output)
return output