[Perf] Remove redundant op for GLM 5.2 (#47198)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -713,6 +713,7 @@ class Indexer(nn.Module):
|
||||
)
|
||||
|
||||
self.is_inplace_rope = is_inplace_rope
|
||||
self.n_head_scale = self.n_head**-0.5
|
||||
self.use_fused_indexer_q = (
|
||||
current_platform.is_cuda()
|
||||
and self.quant_block_size == self.head_dim
|
||||
@@ -761,15 +762,16 @@ class Indexer(nn.Module):
|
||||
rotary_emb.cos_sin_cache,
|
||||
weights,
|
||||
self.softmax_scale,
|
||||
self.n_head**-0.5,
|
||||
self.n_head_scale,
|
||||
rotary_emb.is_neox_style,
|
||||
)
|
||||
|
||||
# rotate only the MQA K
|
||||
q_dummy = torch.empty_like(k_pe.unsqueeze(1))
|
||||
_, k_pe = rotary_emb(positions, q_dummy, k_pe.unsqueeze(1))
|
||||
k_pe = k_pe.reshape(-1, 1, self.rope_dim)
|
||||
k = torch.cat([k_pe.squeeze(-2), k_nope], dim=-1)
|
||||
k_pe = k_pe.unsqueeze(1)
|
||||
q_dummy = torch.empty_like(k_pe)
|
||||
_, k_pe = rotary_emb(positions, q_dummy, k_pe)
|
||||
k_pe = k_pe.reshape(-1, self.rope_dim)
|
||||
k = torch.cat([k_pe, k_nope], dim=-1)
|
||||
|
||||
return self.indexer_op(hidden_states, q_fp8, k, weights)
|
||||
else:
|
||||
@@ -790,13 +792,13 @@ class Indexer(nn.Module):
|
||||
# Note: RoPE (NeoX) can introduce extra leading dimensions during
|
||||
# compilation so we need to reshape back to token-flattened shapes
|
||||
q_pe = q_pe.reshape(-1, self.n_head, self.rope_dim)
|
||||
k_pe = k_pe.reshape(-1, 1, self.rope_dim)
|
||||
k_pe = k_pe.reshape(-1, self.rope_dim)
|
||||
|
||||
# `rotary_emb` is shape-preserving; `q_pe` is already
|
||||
# [num_tokens, n_head, rope_dim].
|
||||
q = torch.cat([q_pe, q_nope], dim=-1)
|
||||
# `k_pe` is [num_tokens, 1, rope_dim] (MQA).
|
||||
k = torch.cat([k_pe.squeeze(-2), k_nope], dim=-1)
|
||||
# `k_pe` is [num_tokens, rope_dim] (MQA).
|
||||
k = torch.cat([k_pe, k_nope], dim=-1)
|
||||
|
||||
# we only quant q here since k quant is fused with cache insertion
|
||||
q = q.view(-1, self.head_dim)
|
||||
@@ -807,12 +809,9 @@ class Indexer(nn.Module):
|
||||
use_ue8m0=self.scale_fmt is not None,
|
||||
)
|
||||
q_fp8 = q_fp8.view(-1, self.n_head, self.head_dim)
|
||||
q_scale = q_scale.view(-1, self.n_head, 1)
|
||||
q_scale = q_scale.view(-1, self.n_head)
|
||||
|
||||
weights = (
|
||||
weights.unsqueeze(-1) * q_scale * self.softmax_scale * self.n_head**-0.5
|
||||
)
|
||||
weights = weights.squeeze(-1)
|
||||
weights = weights * q_scale * self.softmax_scale * self.n_head_scale
|
||||
|
||||
return self.indexer_op(hidden_states, q_fp8, k, weights)
|
||||
|
||||
@@ -1287,13 +1286,10 @@ class DeepseekV2DecoderLayer(nn.Module):
|
||||
hidden_states = tensor_model_parallel_all_gather(hidden_states, 0)
|
||||
hidden_states = hidden_states[:full_num_tokens]
|
||||
|
||||
attn_kwargs = {
|
||||
"positions": positions,
|
||||
"hidden_states": hidden_states,
|
||||
}
|
||||
if not self.use_mha:
|
||||
attn_kwargs["llama_4_scaling"] = llama_4_scaling
|
||||
hidden_states = self.self_attn(**attn_kwargs)
|
||||
if self.use_mha:
|
||||
hidden_states = self.self_attn(positions, hidden_states)
|
||||
else:
|
||||
hidden_states = self.self_attn(positions, hidden_states, llama_4_scaling)
|
||||
|
||||
if (
|
||||
not isinstance(self.self_attn, DeepseekAttention)
|
||||
|
||||
Reference in New Issue
Block a user