diff --git a/vllm/model_executor/layers/mhc.py b/vllm/model_executor/layers/mhc.py index fd9d287e9d..23733f7693 100644 --- a/vllm/model_executor/layers/mhc.py +++ b/vllm/model_executor/layers/mhc.py @@ -5,6 +5,7 @@ import torch # this import will also register the custom ops # import vllm.model_executor.kernels.mhc # noqa: F401 import vllm.model_executor.kernels.mhc as mhc_kernels +from vllm._aiter_ops import is_aiter_found_and_supported from vllm.model_executor.custom_op import CustomOp from vllm.platforms import current_platform from vllm.utils.import_utils import has_tilelang @@ -25,6 +26,7 @@ def _has_tilelang_mhc() -> bool: HAS_TILELANG_MHC = _has_tilelang_mhc() +HAS_AITER_MHC = is_aiter_found_and_supported() # --8<-- [start:mhc_pre] @@ -87,25 +89,20 @@ class MHCPreOp(CustomOp): norm_weight: torch.Tensor | None = None, norm_eps: float = 0.0, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - # TODO: Reenable aiter after we are at the aiter - # version that has this bugfix - # https://github.com/ROCm/aiter/commit/b639cb63bcac4672dce33a731fad042a65cb3649 - # It has accuracy problem at large number of tokens. - # hidden_size = residual.shape[-1] - # if hidden_size % 256 == 0: - # return torch.ops.vllm.mhc_pre_aiter( - # residual, - # fn, - # hc_scale, - # hc_base, - # rms_eps, - # hc_pre_eps, - # hc_sinkhorn_eps, - # hc_post_mult_value, - # sinkhorn_repeat, - # ) - # else: - if HAS_TILELANG_MHC: + hidden_size = residual.shape[-1] + if HAS_AITER_MHC and hidden_size % 256 == 0: + return torch.ops.vllm.mhc_pre_aiter( + residual, + fn, + hc_scale, + hc_base, + rms_eps, + hc_pre_eps, + hc_sinkhorn_eps, + hc_post_mult_value, + sinkhorn_repeat, + ) + elif HAS_TILELANG_MHC: return torch.ops.vllm.mhc_pre_tilelang( residual, fn, @@ -227,19 +224,14 @@ class MHCPostOp(CustomOp): post_layer_mix: torch.Tensor, comb_res_mix: torch.Tensor, ) -> torch.Tensor: - # TODO: Reenable aiter after we are at the aiter - # version that has this bugfix - # https://github.com/ROCm/aiter/commit/b639cb63bcac4672dce33a731fad042a65cb3649 - # It has accuracy problem at large number of tokens. - # hidden_size = residual.shape[-1] - # if hidden_size % 256 == 0: - # return torch.ops.vllm.mhc_post_aiter( - # x, - # residual, - # post_layer_mix, - # comb_res_mix, - # ) - # else: + hidden_size = residual.shape[-1] + if HAS_AITER_MHC and hidden_size % 256 == 0: + return torch.ops.vllm.mhc_post_aiter( + x, + residual, + post_layer_mix, + comb_res_mix, + ) if HAS_TILELANG_MHC: return torch.ops.vllm.mhc_post_tilelang( x, residual, post_layer_mix, comb_res_mix diff --git a/vllm/models/deepseek_v4/amd/model.py b/vllm/models/deepseek_v4/amd/model.py index edb9235115..01646ad994 100644 --- a/vllm/models/deepseek_v4/amd/model.py +++ b/vllm/models/deepseek_v4/amd/model.py @@ -27,6 +27,7 @@ from vllm.model_executor.layers.linear import ( ) from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.mhc import ( + HAS_AITER_MHC, HAS_TILELANG_MHC, HCHeadOp, MHCFusedPostPreOp, @@ -303,7 +304,9 @@ class DeepseekV4DecoderLayer(nn.Module): self.mhc_pre = MHCPreOp() self.mhc_post = MHCPostOp() self.mhc_fused_post_pre = MHCFusedPostPreOp() - self.has_tilelang = HAS_TILELANG_MHC + self.use_fused_mhc = HAS_TILELANG_MHC and not ( + HAS_AITER_MHC and self.hidden_size % 256 == 0 + ) def hc_pre( self, @@ -425,7 +428,7 @@ class DeepseekV4DecoderLayer(nn.Module): ) -> tuple[ torch.Tensor, torch.Tensor | None, torch.Tensor | None, torch.Tensor | None ]: - if not self.has_tilelang: + if not self.use_fused_mhc: return self._forward_unfused_post_pre( x, positions, input_ids, post_mix, res_mix, residual ) @@ -513,7 +516,6 @@ class DeepseekV4Model(nn.Module): requires_grad=False, ) self.hc_head_op = HCHeadOp() - self.has_tilelang = HAS_TILELANG_MHC # Pre-hc_head residual stream buffer for the MTP draft. Stable # address (outside the cudagraph pool) so the copy_ in forward() # refreshes it correctly across captured shapes. @@ -580,7 +582,7 @@ class DeepseekV4Model(nn.Module): res_mix, residual, ) - if layer is not None and self.has_tilelang: + if layer is not None and layer.use_fused_mhc: hidden_states = layer.hc_post(hidden_states, residual, post_mix, res_mix) if not get_pp_group().is_last_rank: diff --git a/vllm/models/deepseek_v4/amd/mtp.py b/vllm/models/deepseek_v4/amd/mtp.py index 5757035cb6..f5ef4bb067 100644 --- a/vllm/models/deepseek_v4/amd/mtp.py +++ b/vllm/models/deepseek_v4/amd/mtp.py @@ -28,7 +28,7 @@ from vllm.model_executor.layers.fused_moe import fused_moe_make_expert_params_ma from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor -from vllm.model_executor.layers.mhc import HAS_TILELANG_MHC, HCHeadOp +from vllm.model_executor.layers.mhc import HCHeadOp from vllm.model_executor.layers.vocab_parallel_embedding import ( VocabParallelEmbedding, ) @@ -123,7 +123,6 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module): ) self.hc_head_op = HCHeadOp() - self.has_tilelang = HAS_TILELANG_MHC def forward( self, @@ -156,7 +155,7 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module): hidden_states, residual, post_mix, res_mix = self.mtp_block( positions=positions, x=hidden_states, input_ids=None ) - if self.has_tilelang: + if self.mtp_block.use_fused_mhc: hidden_states = self.mtp_block.hc_post( hidden_states, residual, post_mix, res_mix )