From 2a9113f9985f10d2089a55e419c3b67b1b9731de Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Sat, 4 Jul 2026 13:25:02 -0400 Subject: [PATCH] [Perf] Remove redundant op for GLM 5.2 (#47198) Signed-off-by: yewentao256 --- vllm/model_executor/models/deepseek_v2.py | 36 ++++++++++------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/vllm/model_executor/models/deepseek_v2.py b/vllm/model_executor/models/deepseek_v2.py index b20db76692..a829c7422e 100644 --- a/vllm/model_executor/models/deepseek_v2.py +++ b/vllm/model_executor/models/deepseek_v2.py @@ -713,6 +713,7 @@ class Indexer(nn.Module): ) self.is_inplace_rope = is_inplace_rope + self.n_head_scale = self.n_head**-0.5 self.use_fused_indexer_q = ( current_platform.is_cuda() and self.quant_block_size == self.head_dim @@ -761,15 +762,16 @@ class Indexer(nn.Module): rotary_emb.cos_sin_cache, weights, self.softmax_scale, - self.n_head**-0.5, + self.n_head_scale, rotary_emb.is_neox_style, ) # rotate only the MQA K - q_dummy = torch.empty_like(k_pe.unsqueeze(1)) - _, k_pe = rotary_emb(positions, q_dummy, k_pe.unsqueeze(1)) - k_pe = k_pe.reshape(-1, 1, self.rope_dim) - k = torch.cat([k_pe.squeeze(-2), k_nope], dim=-1) + k_pe = k_pe.unsqueeze(1) + q_dummy = torch.empty_like(k_pe) + _, k_pe = rotary_emb(positions, q_dummy, k_pe) + k_pe = k_pe.reshape(-1, self.rope_dim) + k = torch.cat([k_pe, k_nope], dim=-1) return self.indexer_op(hidden_states, q_fp8, k, weights) else: @@ -790,13 +792,13 @@ class Indexer(nn.Module): # Note: RoPE (NeoX) can introduce extra leading dimensions during # compilation so we need to reshape back to token-flattened shapes q_pe = q_pe.reshape(-1, self.n_head, self.rope_dim) - k_pe = k_pe.reshape(-1, 1, self.rope_dim) + k_pe = k_pe.reshape(-1, self.rope_dim) # `rotary_emb` is shape-preserving; `q_pe` is already # [num_tokens, n_head, rope_dim]. q = torch.cat([q_pe, q_nope], dim=-1) - # `k_pe` is [num_tokens, 1, rope_dim] (MQA). - k = torch.cat([k_pe.squeeze(-2), k_nope], dim=-1) + # `k_pe` is [num_tokens, rope_dim] (MQA). + k = torch.cat([k_pe, k_nope], dim=-1) # we only quant q here since k quant is fused with cache insertion q = q.view(-1, self.head_dim) @@ -807,12 +809,9 @@ class Indexer(nn.Module): use_ue8m0=self.scale_fmt is not None, ) q_fp8 = q_fp8.view(-1, self.n_head, self.head_dim) - q_scale = q_scale.view(-1, self.n_head, 1) + q_scale = q_scale.view(-1, self.n_head) - weights = ( - weights.unsqueeze(-1) * q_scale * self.softmax_scale * self.n_head**-0.5 - ) - weights = weights.squeeze(-1) + weights = weights * q_scale * self.softmax_scale * self.n_head_scale return self.indexer_op(hidden_states, q_fp8, k, weights) @@ -1287,13 +1286,10 @@ class DeepseekV2DecoderLayer(nn.Module): hidden_states = tensor_model_parallel_all_gather(hidden_states, 0) hidden_states = hidden_states[:full_num_tokens] - attn_kwargs = { - "positions": positions, - "hidden_states": hidden_states, - } - if not self.use_mha: - attn_kwargs["llama_4_scaling"] = llama_4_scaling - hidden_states = self.self_attn(**attn_kwargs) + if self.use_mha: + hidden_states = self.self_attn(positions, hidden_states) + else: + hidden_states = self.self_attn(positions, hidden_states, llama_4_scaling) if ( not isinstance(self.self_attn, DeepseekAttention)