[ROCm][Deepseek] dsv3.2 further optimization (#41217)
Signed-off-by: ganyi <ygan@amd.com> Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com> Co-authored-by: Matthew Wong <Matthew.Wong2@amd.com>
This commit is contained in:
co-authored by
Matthew Wong
parent
c3e64696cd
commit
bc635fad23
@@ -674,30 +674,45 @@ class Indexer(nn.Module):
|
||||
) -> torch.Tensor:
|
||||
q, _ = self.wq_b(qr)
|
||||
q = q.view(-1, self.n_head, self.head_dim)
|
||||
q_pe, q_nope = torch.split(
|
||||
q, [self.rope_dim, self.head_dim - self.rope_dim], dim=-1
|
||||
)
|
||||
# Fused wk + weights_proj: one GEMM, then split
|
||||
kw, _ = self.wk_weights_proj(hidden_states)
|
||||
k = kw[:, : self.head_dim]
|
||||
weights = kw[:, self.head_dim :]
|
||||
|
||||
k = self.k_norm(k)
|
||||
k_pe, k_nope = torch.split(
|
||||
k, [self.rope_dim, self.head_dim - self.rope_dim], dim=-1
|
||||
)
|
||||
if current_platform.is_rocm():
|
||||
# This path should works on all platform, will remove extra
|
||||
# branches in the future
|
||||
# Fused wk + weights_proj: one GEMM, then split
|
||||
kw, _ = self.wk_weights_proj(hidden_states)
|
||||
k = kw[:, : self.head_dim]
|
||||
weights = kw[:, self.head_dim :]
|
||||
|
||||
q_pe, k_pe = rotary_emb(positions, q_pe, k_pe.unsqueeze(1))
|
||||
# Note: RoPE (NeoX) can introduce extra leading dimensions during compilation
|
||||
# so we need to reshape back to token-flattened shapes
|
||||
q_pe = q_pe.reshape(-1, self.n_head, self.rope_dim)
|
||||
k_pe = k_pe.reshape(-1, 1, self.rope_dim)
|
||||
k = self.k_norm(k)
|
||||
|
||||
# `rotary_emb` is shape-preserving; `q_pe` is already
|
||||
# [num_tokens, n_head, rope_dim].
|
||||
q = torch.cat([q_pe, q_nope], dim=-1)
|
||||
# `k_pe` is [num_tokens, 1, rope_dim] (MQA).
|
||||
k = torch.cat([k_pe.squeeze(-2), k_nope], dim=-1)
|
||||
rotary_emb(
|
||||
positions, q[..., : self.rope_dim], k[..., : self.rope_dim].unsqueeze(1)
|
||||
)
|
||||
else:
|
||||
q_pe, q_nope = torch.split(
|
||||
q, [self.rope_dim, self.head_dim - self.rope_dim], dim=-1
|
||||
)
|
||||
# Fused wk + weights_proj: one GEMM, then split
|
||||
kw, _ = self.wk_weights_proj(hidden_states)
|
||||
k = kw[:, : self.head_dim]
|
||||
weights = kw[:, self.head_dim :]
|
||||
|
||||
k = self.k_norm(k)
|
||||
k_pe, k_nope = torch.split(
|
||||
k, [self.rope_dim, self.head_dim - self.rope_dim], dim=-1
|
||||
)
|
||||
|
||||
q_pe, k_pe = rotary_emb(positions, q_pe, k_pe.unsqueeze(1))
|
||||
# Note: RoPE (NeoX) can introduce extra leading dimensions during
|
||||
# compilation so we need to reshape back to token-flattened shapes
|
||||
q_pe = q_pe.reshape(-1, self.n_head, self.rope_dim)
|
||||
k_pe = k_pe.reshape(-1, 1, self.rope_dim)
|
||||
|
||||
# `rotary_emb` is shape-preserving; `q_pe` is already
|
||||
# [num_tokens, n_head, rope_dim].
|
||||
q = torch.cat([q_pe, q_nope], dim=-1)
|
||||
# `k_pe` is [num_tokens, 1, rope_dim] (MQA).
|
||||
k = torch.cat([k_pe.squeeze(-2), k_nope], dim=-1)
|
||||
|
||||
# we only quant q here since k quant is fused with cache insertion
|
||||
q = q.view(-1, self.head_dim)
|
||||
|
||||
Reference in New Issue
Block a user