[Bugfix][ROCm] Fix Accuracy Drop in Sparse Indexer on gfx950 (#43781)
Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com> Signed-off-by: vllmellm <vllm.ellm@embeddedllm.com> Signed-off-by: kliuae <kuanfu.liu@embeddedllm.com> Co-authored-by: tjtanaa <tunjian.tan@embeddedllm.com> Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com>
This commit is contained in:
co-authored by
tjtanaa
vllmellm
parent
4ec2817313
commit
a583c84e2b
@@ -612,6 +612,7 @@ class Indexer(nn.Module):
|
||||
cache_config: CacheConfig | None,
|
||||
topk_indices_buffer: torch.Tensor | None,
|
||||
prefix: str = "",
|
||||
is_inplace_rope: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
self.vllm_config = vllm_config
|
||||
@@ -673,15 +674,21 @@ class Indexer(nn.Module):
|
||||
self.topk_indices_buffer,
|
||||
)
|
||||
|
||||
self.is_inplace_rope = is_inplace_rope
|
||||
|
||||
def forward(
|
||||
self, hidden_states: torch.Tensor, qr: torch.Tensor, positions, rotary_emb
|
||||
) -> torch.Tensor:
|
||||
q, _ = self.wq_b(qr)
|
||||
q = q.view(-1, self.n_head, self.head_dim)
|
||||
|
||||
if current_platform.is_rocm():
|
||||
if current_platform.is_rocm() and self.is_inplace_rope:
|
||||
# This path should works on all platform, will remove extra
|
||||
# branches in the future
|
||||
# This fast path relies on rotary_emb mutating q and k inplace.
|
||||
# On ROCm, this is only valid for kernels used as custom ops.
|
||||
# In pytorch-native rope for inductor fusion, rotated q/k tensors
|
||||
# are not mutated inplace but returned as new tensors.
|
||||
# Fused wk + weights_proj: one GEMM, then split
|
||||
kw, _ = self.wk_weights_proj(hidden_states)
|
||||
k = kw[:, : self.head_dim]
|
||||
@@ -1002,6 +1009,7 @@ class DeepseekV2MLAAttention(nn.Module):
|
||||
cache_config,
|
||||
topk_indices_buffer,
|
||||
f"{prefix}.indexer",
|
||||
is_inplace_rope=self.indexer_rope_emb.enabled(),
|
||||
)
|
||||
|
||||
# Enable IndexCache for DeepSeek models to reduce redundant top-k
|
||||
|
||||
Reference in New Issue
Block a user