[Bugfix][ROCm] Fix Accuracy Drop in Sparse Indexer on gfx950 (#43781)

Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
Signed-off-by: vllmellm <vllm.ellm@embeddedllm.com>
Signed-off-by: kliuae <kuanfu.liu@embeddedllm.com>
Co-authored-by: tjtanaa <tunjian.tan@embeddedllm.com>
Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com>
This commit is contained in:
kliuae
2026-05-28 03:37:15 -07:00
committed by GitHub
co-authored by tjtanaa vllmellm
parent 4ec2817313
commit a583c84e2b
2 changed files with 14 additions and 4 deletions
+9 -1
View File
@@ -612,6 +612,7 @@ class Indexer(nn.Module):
cache_config: CacheConfig | None,
topk_indices_buffer: torch.Tensor | None,
prefix: str = "",
is_inplace_rope: bool = False,
):
super().__init__()
self.vllm_config = vllm_config
@@ -673,15 +674,21 @@ class Indexer(nn.Module):
self.topk_indices_buffer,
)
self.is_inplace_rope = is_inplace_rope
def forward(
self, hidden_states: torch.Tensor, qr: torch.Tensor, positions, rotary_emb
) -> torch.Tensor:
q, _ = self.wq_b(qr)
q = q.view(-1, self.n_head, self.head_dim)
if current_platform.is_rocm():
if current_platform.is_rocm() and self.is_inplace_rope:
# This path should works on all platform, will remove extra
# branches in the future
# This fast path relies on rotary_emb mutating q and k inplace.
# On ROCm, this is only valid for kernels used as custom ops.
# In pytorch-native rope for inductor fusion, rotated q/k tensors
# are not mutated inplace but returned as new tensors.
# Fused wk + weights_proj: one GEMM, then split
kw, _ = self.wk_weights_proj(hidden_states)
k = kw[:, : self.head_dim]
@@ -1002,6 +1009,7 @@ class DeepseekV2MLAAttention(nn.Module):
cache_config,
topk_indices_buffer,
f"{prefix}.indexer",
is_inplace_rope=self.indexer_rope_emb.enabled(),
)
# Enable IndexCache for DeepSeek models to reduce redundant top-k