[Perf] RMSNorm uncontiguous support, 1.2~3.1x kernel performance improvement (#49750)
Signed-off-by: yewentao256 <zhyanwentao@126.com> Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
@@ -211,8 +211,8 @@ class ApertusAttention(nn.Module):
|
||||
) -> torch.Tensor:
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
q = self.q_norm(q.contiguous().view(-1, self.head_dim)).view_as(q)
|
||||
k = self.k_norm(k.contiguous().view(-1, self.head_dim)).view_as(k)
|
||||
q = self.q_norm(q.view(-1, self.num_heads, self.head_dim)).view_as(q)
|
||||
k = self.k_norm(k.view(-1, self.num_kv_heads, self.head_dim)).view_as(k)
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
attn_output = self.attn(q, k, v)
|
||||
output, _ = self.o_proj(attn_output)
|
||||
|
||||
@@ -1481,8 +1481,6 @@ class DeepseekV2Model(nn.Module):
|
||||
hidden_states, residual = combined_states.split(
|
||||
[self.hidden_size, self.hidden_size], dim=-1
|
||||
)
|
||||
# fused_add_rms_norm requires a contiguous residual
|
||||
residual = residual.contiguous()
|
||||
if idx in self.aux_hidden_state_layers:
|
||||
aux_hidden_state = hidden_states + residual
|
||||
if aux_hidden_state.shape[0] != positions.shape[0]:
|
||||
@@ -1507,8 +1505,6 @@ class DeepseekV2Model(nn.Module):
|
||||
hidden_states, residual = combined_states.split(
|
||||
[self.hidden_size, self.hidden_size], dim=-1
|
||||
)
|
||||
# fused_add_rms_norm requires a contiguous residual
|
||||
residual = residual.contiguous()
|
||||
|
||||
if self.end_layer in self.aux_hidden_state_layers:
|
||||
aux_hidden_states.append(hidden_states + residual)
|
||||
|
||||
@@ -238,10 +238,10 @@ class HunYuanAttention(nn.Module):
|
||||
ori_k = k
|
||||
if self.use_qk_norm:
|
||||
q = self.query_layernorm(
|
||||
q.view(-1, self.num_heads, self.head_dim).contiguous()
|
||||
q.view(-1, self.num_heads, self.head_dim),
|
||||
)
|
||||
k = self.key_layernorm(
|
||||
k.view(-1, self.num_kv_heads, self.head_dim).contiguous()
|
||||
k.view(-1, self.num_kv_heads, self.head_dim),
|
||||
)
|
||||
|
||||
attn_output = self.attn(q, k, v)
|
||||
@@ -346,10 +346,10 @@ class HunYuanCrossAttention(nn.Module):
|
||||
q, _ = self.rotary_emb(positions, q, k_tmp)
|
||||
if self.use_qk_norm:
|
||||
q = self.query_layernorm(
|
||||
q.view(-1, self.num_heads, self.head_dim).contiguous()
|
||||
q.view(-1, self.num_heads, self.head_dim),
|
||||
)
|
||||
k = self.key_layernorm(
|
||||
k.view(-1, self.num_kv_heads, self.head_dim).contiguous()
|
||||
k.view(-1, self.num_kv_heads, self.head_dim),
|
||||
)
|
||||
|
||||
attn_output = self.attn(q, k, v)
|
||||
|
||||
@@ -169,8 +169,8 @@ class Lfm2Attention(nn.Module):
|
||||
n_tokens, _ = hidden_states.shape
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
q = q.view(n_tokens, self.num_heads, self.head_dim).contiguous()
|
||||
k = k.view(n_tokens, self.num_kv_heads, self.head_dim).contiguous()
|
||||
q = q.view(n_tokens, self.num_heads, self.head_dim)
|
||||
k = k.view(n_tokens, self.num_kv_heads, self.head_dim)
|
||||
q = self.q_layernorm(q)
|
||||
k = self.k_layernorm(k)
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
|
||||
@@ -255,8 +255,8 @@ class Lfm2MoeAttention(nn.Module):
|
||||
n_tokens, _ = hidden_states.shape
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
q = q.view(n_tokens, self.num_heads, self.head_dim).contiguous()
|
||||
k = k.view(n_tokens, self.num_kv_heads, self.head_dim).contiguous()
|
||||
q = q.view(n_tokens, self.num_heads, self.head_dim)
|
||||
k = k.view(n_tokens, self.num_kv_heads, self.head_dim)
|
||||
q = self.q_layernorm(q)
|
||||
k = self.k_layernorm(k)
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
|
||||
@@ -146,7 +146,7 @@ class MiniCPM3Attention(nn.Module):
|
||||
latent_cache, _ = self.kv_a_proj_with_mqa(hidden_states)
|
||||
kv_a, _ = latent_cache.split([self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
|
||||
latent_cache = latent_cache.unsqueeze(1)
|
||||
kv_a = self.kv_a_layernorm(kv_a.contiguous())
|
||||
kv_a = self.kv_a_layernorm(kv_a)
|
||||
kv, _ = self.kv_b_proj(kv_a)
|
||||
kv = kv.view(-1, self.num_local_heads, self.qk_nope_head_dim + self.v_head_dim)
|
||||
k_nope, v = kv.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
|
||||
|
||||
@@ -271,11 +271,11 @@ class Step3p5Attention(nn.Module):
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
# Add qk-norm inline similar to Qwen3 MOE attention
|
||||
q_by_head = q.view(*q.shape[:-1], q.shape[-1] // self.head_dim, self.head_dim)
|
||||
q_by_head = self.q_norm(q_by_head.contiguous())
|
||||
q_by_head = self.q_norm(q_by_head)
|
||||
q = q_by_head.view(q.shape)
|
||||
|
||||
k_by_head = k.view(*k.shape[:-1], k.shape[-1] // self.head_dim, self.head_dim)
|
||||
k_by_head = self.k_norm(k_by_head.contiguous())
|
||||
k_by_head = self.k_norm(k_by_head)
|
||||
k = k_by_head.view(k.shape)
|
||||
if self.use_rope:
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
|
||||
Reference in New Issue
Block a user