[ROCm] [Bugfix] Fix DeepSeek V4 Functionality and Accuracy (#42810)
Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
This commit is contained in:
@@ -61,31 +61,35 @@ class MHCPreOp(CustomOp):
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
hidden_size = residual.shape[-1]
|
||||
if hidden_size % 256 == 0:
|
||||
return torch.ops.vllm.mhc_pre_aiter(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
)
|
||||
else:
|
||||
return mhc_kernels.mhc_pre_torch(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
)
|
||||
# TODO: Reenable aiter after we are at the aiter
|
||||
# version that has this bugfix
|
||||
# https://github.com/ROCm/aiter/commit/b639cb63bcac4672dce33a731fad042a65cb3649
|
||||
# It has accuracy problem at large number of tokens.
|
||||
# hidden_size = residual.shape[-1]
|
||||
# if hidden_size % 256 == 0:
|
||||
# return torch.ops.vllm.mhc_pre_aiter(
|
||||
# residual,
|
||||
# fn,
|
||||
# hc_scale,
|
||||
# hc_base,
|
||||
# rms_eps,
|
||||
# hc_pre_eps,
|
||||
# hc_sinkhorn_eps,
|
||||
# hc_post_mult_value,
|
||||
# sinkhorn_repeat,
|
||||
# )
|
||||
# else:
|
||||
return mhc_kernels.mhc_pre_torch(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
)
|
||||
|
||||
def forward_native(self, *args, **kwargs):
|
||||
raise NotImplementedError("Native implementation of mhc_pre is not available")
|
||||
@@ -124,21 +128,25 @@ class MHCPostOp(CustomOp):
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
hidden_size = residual.shape[-1]
|
||||
if hidden_size % 256 == 0:
|
||||
return torch.ops.vllm.mhc_post_aiter(
|
||||
x,
|
||||
residual,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
)
|
||||
else:
|
||||
return mhc_kernels.mhc_post_torch(
|
||||
x,
|
||||
residual,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
)
|
||||
# TODO: Reenable aiter after we are at the aiter
|
||||
# version that has this bugfix
|
||||
# https://github.com/ROCm/aiter/commit/b639cb63bcac4672dce33a731fad042a65cb3649
|
||||
# It has accuracy problem at large number of tokens.
|
||||
# hidden_size = residual.shape[-1]
|
||||
# if hidden_size % 256 == 0:
|
||||
# return torch.ops.vllm.mhc_post_aiter(
|
||||
# x,
|
||||
# residual,
|
||||
# post_layer_mix,
|
||||
# comb_res_mix,
|
||||
# )
|
||||
# else:
|
||||
return mhc_kernels.mhc_post_torch(
|
||||
x,
|
||||
residual,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
)
|
||||
|
||||
def forward_native(self, *args, **kwargs):
|
||||
raise NotImplementedError("Native implementation of mhc_post is not available")
|
||||
|
||||
@@ -505,27 +505,6 @@ class SparseAttnIndexer(CustomOp):
|
||||
assert isinstance(q_quant, torch.Tensor), (
|
||||
"AMD sparse_attn_indexer expects a single FP8 q_quant tensor"
|
||||
)
|
||||
if self.skip_k_cache_insert or not rocm_aiter_ops.is_enabled():
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
rocm_aiter_sparse_attn_indexer_native,
|
||||
)
|
||||
|
||||
return rocm_aiter_sparse_attn_indexer_native(
|
||||
hidden_states,
|
||||
_encode_layer_name(self.k_cache.prefix),
|
||||
self.k_cache.kv_cache,
|
||||
q_quant,
|
||||
k,
|
||||
weights,
|
||||
self.quant_block_size,
|
||||
self.scale_fmt,
|
||||
self.topk_tokens,
|
||||
self.head_dim,
|
||||
self.max_model_len,
|
||||
self.max_total_seq_len,
|
||||
self.topk_indices_buffer,
|
||||
skip_k_cache_insert=self.skip_k_cache_insert,
|
||||
)
|
||||
if rocm_aiter_ops.is_enabled():
|
||||
return torch.ops.vllm.rocm_aiter_sparse_attn_indexer(
|
||||
hidden_states,
|
||||
@@ -541,5 +520,9 @@ class SparseAttnIndexer(CustomOp):
|
||||
self.max_model_len,
|
||||
self.max_total_seq_len,
|
||||
self.topk_indices_buffer,
|
||||
skip_k_cache_insert=self.skip_k_cache_insert,
|
||||
)
|
||||
raise RuntimeError("Sparse attention indexer ROCm path could not be selected.")
|
||||
raise RuntimeError(
|
||||
"Sparse attention indexer ROCm path is only supported on AITER. "
|
||||
"Please enable aiter with VLLM_ROCM_USE_AITER=1"
|
||||
)
|
||||
|
||||
@@ -1277,7 +1277,8 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
x, post, comb = self.hc_pre(
|
||||
x, self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base
|
||||
)
|
||||
x = self.ffn_norm(x)
|
||||
# ffn_norm is now folded into self.ffn.norm_gate; ffn() takes
|
||||
# the pre-norm activation directly.
|
||||
x = self.ffn(x, input_ids)
|
||||
x = self.hc_post(x, residual, post, comb)
|
||||
return x, None, None, None
|
||||
|
||||
Reference in New Issue
Block a user