[ROCm] [Bugfix] Fix DeepSeek V4 Functionality and Accuracy (#42810)

Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
This commit is contained in:
TJian
2026-05-17 12:18:50 -04:00
committed by GitHub
parent 1c8e9c0399
commit 599e75f432
4 changed files with 88 additions and 177 deletions
+48 -40
View File
@@ -61,31 +61,35 @@ class MHCPreOp(CustomOp):
sinkhorn_repeat: int,
n_splits: int = 1,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
hidden_size = residual.shape[-1]
if hidden_size % 256 == 0:
return torch.ops.vllm.mhc_pre_aiter(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
)
else:
return mhc_kernels.mhc_pre_torch(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
)
# TODO: Reenable aiter after we are at the aiter
# version that has this bugfix
# https://github.com/ROCm/aiter/commit/b639cb63bcac4672dce33a731fad042a65cb3649
# It has accuracy problem at large number of tokens.
# hidden_size = residual.shape[-1]
# if hidden_size % 256 == 0:
# return torch.ops.vllm.mhc_pre_aiter(
# residual,
# fn,
# hc_scale,
# hc_base,
# rms_eps,
# hc_pre_eps,
# hc_sinkhorn_eps,
# hc_post_mult_value,
# sinkhorn_repeat,
# )
# else:
return mhc_kernels.mhc_pre_torch(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
)
def forward_native(self, *args, **kwargs):
raise NotImplementedError("Native implementation of mhc_pre is not available")
@@ -124,21 +128,25 @@ class MHCPostOp(CustomOp):
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
) -> torch.Tensor:
hidden_size = residual.shape[-1]
if hidden_size % 256 == 0:
return torch.ops.vllm.mhc_post_aiter(
x,
residual,
post_layer_mix,
comb_res_mix,
)
else:
return mhc_kernels.mhc_post_torch(
x,
residual,
post_layer_mix,
comb_res_mix,
)
# TODO: Reenable aiter after we are at the aiter
# version that has this bugfix
# https://github.com/ROCm/aiter/commit/b639cb63bcac4672dce33a731fad042a65cb3649
# It has accuracy problem at large number of tokens.
# hidden_size = residual.shape[-1]
# if hidden_size % 256 == 0:
# return torch.ops.vllm.mhc_post_aiter(
# x,
# residual,
# post_layer_mix,
# comb_res_mix,
# )
# else:
return mhc_kernels.mhc_post_torch(
x,
residual,
post_layer_mix,
comb_res_mix,
)
def forward_native(self, *args, **kwargs):
raise NotImplementedError("Native implementation of mhc_post is not available")
@@ -505,27 +505,6 @@ class SparseAttnIndexer(CustomOp):
assert isinstance(q_quant, torch.Tensor), (
"AMD sparse_attn_indexer expects a single FP8 q_quant tensor"
)
if self.skip_k_cache_insert or not rocm_aiter_ops.is_enabled():
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
rocm_aiter_sparse_attn_indexer_native,
)
return rocm_aiter_sparse_attn_indexer_native(
hidden_states,
_encode_layer_name(self.k_cache.prefix),
self.k_cache.kv_cache,
q_quant,
k,
weights,
self.quant_block_size,
self.scale_fmt,
self.topk_tokens,
self.head_dim,
self.max_model_len,
self.max_total_seq_len,
self.topk_indices_buffer,
skip_k_cache_insert=self.skip_k_cache_insert,
)
if rocm_aiter_ops.is_enabled():
return torch.ops.vllm.rocm_aiter_sparse_attn_indexer(
hidden_states,
@@ -541,5 +520,9 @@ class SparseAttnIndexer(CustomOp):
self.max_model_len,
self.max_total_seq_len,
self.topk_indices_buffer,
skip_k_cache_insert=self.skip_k_cache_insert,
)
raise RuntimeError("Sparse attention indexer ROCm path could not be selected.")
raise RuntimeError(
"Sparse attention indexer ROCm path is only supported on AITER. "
"Please enable aiter with VLLM_ROCM_USE_AITER=1"
)