# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import torch from vllm.utils.torch_utils import direct_register_custom_op def mhc_pre_aiter( residual: torch.Tensor, fn: torch.Tensor, hc_scale: torch.Tensor, hc_base: torch.Tensor, rms_eps: float, hc_pre_eps: float, hc_sinkhorn_eps: float, hc_post_mult_value: float, sinkhorn_repeat: int, n_splits: int = 1, norm_weight: torch.Tensor | None = None, norm_eps: float = 0.0, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """Forward pass for mHC pre block. Args: residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16 fn: shape (hc_mult3, hc_mult * hidden_size), dtype torch.float32 hc_scale: shape (3,), dtype torch.float32 hc_base: shape (hc_mult3,), dtype torch.float32 rms_eps: RMS normalization epsilon hc_pre_eps: pre-mix epsilon hc_sinkhorn_eps: sinkhorn epsilon hc_post_mult_value: post-mix multiplier value sinkhorn_repeat: number of sinkhorn iterations n_splits: split-k factor; norm_weight: optional RMSNorm weight fused into the pre kernel norm_eps: epsilon for the fused RMSNorm when norm_weight is set Returns: post_mix: shape (..., hc_mult), dtype torch.float32 comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32 layer_input: shape (..., hidden_size), dtype torch.bfloat16 """ hidden_size = residual.shape[-1] assert hidden_size % 256 == 0 from vllm._aiter_ops import rocm_aiter_ops return rocm_aiter_ops.mhc_pre( residual, fn, hc_scale, hc_base, rms_eps, hc_pre_eps, hc_sinkhorn_eps, hc_post_mult_value, sinkhorn_repeat, norm_weight, norm_eps, ) def _mhc_pre_aiter_fake( residual: torch.Tensor, fn: torch.Tensor, hc_scale: torch.Tensor, hc_base: torch.Tensor, rms_eps: float, hc_pre_eps: float, hc_sinkhorn_eps: float, hc_post_mult_value: float, sinkhorn_repeat: int, n_splits: int = 1, norm_weight: torch.Tensor | None = None, norm_eps: float = 0.0, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: hc_mult = residual.shape[-2] hidden_size = residual.shape[-1] outer_shape = residual.shape[:-2] # Create empty tensors with correct shapes for meta device / shape inference post_mix = torch.empty( *outer_shape, hc_mult, 1, dtype=torch.float32, device=residual.device, ) comb_mix = torch.empty( *outer_shape, hc_mult, hc_mult, dtype=torch.float32, device=residual.device, ) layer_input = torch.empty( *outer_shape, hidden_size, dtype=torch.bfloat16, device=residual.device, ) return post_mix, comb_mix, layer_input def mhc_pre_delayed_aiter( residual: torch.Tensor, fn: torch.Tensor, hc_scale: torch.Tensor, hc_base: torch.Tensor, rms_eps: float, hc_pre_eps: float, hc_sinkhorn_eps: float, hc_post_mult_value: float, sinkhorn_repeat: int, pre_mix: torch.Tensor | None = None, sublayer_out: torch.Tensor | None = None, post_layer_mix: torch.Tensor | None = None, comb_res_mix: torch.Tensor | None = None, residual_out: torch.Tensor | None = None, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: """MHC pre with the pre-mix carried in from the previous sublayer. Matches ``mhc_pre_delayed_torch``: the stream collapse uses *pre_mix* rather than the gate computed here, and that gate is returned as the pre-mix for the next sublayer seam. Args: residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16 fn: shape (hc_mult3, hc_mult * hidden_size), dtype torch.float32 hc_scale: shape (3,), dtype torch.float32 hc_base: shape (hc_mult3,), dtype torch.float32 rms_eps: RMS normalization epsilon hc_pre_eps: pre-mix epsilon hc_sinkhorn_eps: sinkhorn epsilon hc_post_mult_value: post-mix multiplier value sinkhorn_repeat: number of sinkhorn iterations pre_mix: shape (..., hc_mult) from the previous sublayer, or None at model entry to select residual stream zero. sublayer_out: attention or FFN output, shape (..., hidden_size). When given with the two mixes below, the preceding post block is applied here so AITER can fold it into the pre projection. post_layer_mix: shape (..., hc_mult, 1), post gate for that block. comb_res_mix: shape (..., hc_mult, hc_mult), residual comb for it. residual_out: shape (..., hc_mult, hidden_size), written with the post block's new residual. Required exactly when the post is folded in. Returns: post_mix: shape (..., hc_mult, 1), dtype torch.float32 comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32 layer_input: shape (..., hidden_size), dtype torch.bfloat16 next_pre_mix: shape (..., hc_mult), dtype torch.float32 """ hidden_size = residual.shape[-1] assert hidden_size % 256 == 0 from vllm._aiter_ops import rocm_aiter_ops return rocm_aiter_ops.mhc_pre_delayed( residual, fn, hc_scale, hc_base, rms_eps, hc_pre_eps, hc_sinkhorn_eps, hc_post_mult_value, sinkhorn_repeat, pre_mix, sublayer_out, post_layer_mix, comb_res_mix, residual_out, ) def _mhc_pre_delayed_aiter_fake( residual: torch.Tensor, fn: torch.Tensor, hc_scale: torch.Tensor, hc_base: torch.Tensor, rms_eps: float, hc_pre_eps: float, hc_sinkhorn_eps: float, hc_post_mult_value: float, sinkhorn_repeat: int, pre_mix: torch.Tensor | None = None, sublayer_out: torch.Tensor | None = None, post_layer_mix: torch.Tensor | None = None, comb_res_mix: torch.Tensor | None = None, residual_out: torch.Tensor | None = None, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: hc_mult = residual.shape[-2] hidden_size = residual.shape[-1] outer_shape = residual.shape[:-2] return ( torch.empty( *outer_shape, hc_mult, 1, dtype=torch.float32, device=residual.device ), torch.empty( *outer_shape, hc_mult, hc_mult, dtype=torch.float32, device=residual.device ), torch.empty( *outer_shape, hidden_size, dtype=torch.bfloat16, device=residual.device ), torch.empty(*outer_shape, hc_mult, dtype=torch.float32, device=residual.device), ) def mhc_fused_post_pre_delayed_rms_norm_aiter( residual: torch.Tensor, fn: torch.Tensor, hc_scale: torch.Tensor, hc_base: torch.Tensor, rms_eps: float, hc_pre_eps: float, hc_sinkhorn_eps: float, hc_post_mult_value: float, sinkhorn_repeat: int, pre_mix: torch.Tensor | None, sublayer_out: torch.Tensor | None, post_layer_mix: torch.Tensor | None, comb_res_mix: torch.Tensor | None, norm_weight: torch.Tensor, norm_eps: float, residual_out: torch.Tensor | None, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: """:func:`mhc_pre_delayed_aiter` with the RMSNorm of the collapse folded in: ``layer_input`` comes back normalised with ``norm_weight`` / ``norm_eps``.""" from vllm._aiter_ops import rocm_aiter_ops return rocm_aiter_ops.mhc_fused_post_pre_delayed_rms_norm( residual, fn, hc_scale, hc_base, rms_eps, hc_pre_eps, hc_sinkhorn_eps, hc_post_mult_value, sinkhorn_repeat, pre_mix, sublayer_out, post_layer_mix, comb_res_mix, norm_weight, norm_eps, residual_out, ) def _mhc_fused_post_pre_delayed_rms_norm_aiter_fake( residual: torch.Tensor, fn: torch.Tensor, hc_scale: torch.Tensor, hc_base: torch.Tensor, rms_eps: float, hc_pre_eps: float, hc_sinkhorn_eps: float, hc_post_mult_value: float, sinkhorn_repeat: int, pre_mix: torch.Tensor | None, sublayer_out: torch.Tensor | None, post_layer_mix: torch.Tensor | None, comb_res_mix: torch.Tensor | None, norm_weight: torch.Tensor, norm_eps: float, residual_out: torch.Tensor | None, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: return _mhc_pre_delayed_aiter_fake( residual, fn, hc_scale, hc_base, rms_eps, hc_pre_eps, hc_sinkhorn_eps, hc_post_mult_value, sinkhorn_repeat, ) def mhc_post_aiter( x: torch.Tensor, residual: torch.Tensor, post_layer_mix: torch.Tensor, comb_res_mix: torch.Tensor, ) -> torch.Tensor: hidden_size = residual.shape[-1] assert hidden_size % 256 == 0 from vllm._aiter_ops import rocm_aiter_ops return rocm_aiter_ops.mhc_post( x, residual, post_layer_mix, comb_res_mix, ) def _mhc_post_aiter_fake( x: torch.Tensor, residual: torch.Tensor, post_layer_mix: torch.Tensor, comb_res_mix: torch.Tensor, ) -> torch.Tensor: return torch.empty_like(residual) def mhc_fused_post_pre_aiter( x: torch.Tensor, residual: torch.Tensor, post_layer_mix: torch.Tensor, comb_res_mix: torch.Tensor, fn: torch.Tensor, hc_scale: torch.Tensor, hc_base: torch.Tensor, rms_eps: float, hc_pre_eps: float, hc_sinkhorn_eps: float, hc_post_mult_value: float, sinkhorn_repeat: int, n_splits: int = 1, tile_n: int = 1, norm_weight: torch.Tensor | None = None, norm_eps: float = 0.0, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: """Fused mHC post + next mHC pre on ROCm via AITER. Returns residual_cur, post_mix_cur, comb_mix_cur, layer_input_cur. """ hidden_size = residual.shape[-1] assert hidden_size % 256 == 0 from vllm._aiter_ops import rocm_aiter_ops return rocm_aiter_ops.mhc_fused_post_pre( x, residual, post_layer_mix, comb_res_mix, fn, hc_scale, hc_base, rms_eps, hc_pre_eps, hc_sinkhorn_eps, hc_post_mult_value, sinkhorn_repeat, norm_weight, norm_eps, ) def _mhc_fused_post_pre_aiter_fake( x: torch.Tensor, residual: torch.Tensor, post_layer_mix: torch.Tensor, comb_res_mix: torch.Tensor, fn: torch.Tensor, hc_scale: torch.Tensor, hc_base: torch.Tensor, rms_eps: float, hc_pre_eps: float, hc_sinkhorn_eps: float, hc_post_mult_value: float, sinkhorn_repeat: int, n_splits: int = 1, tile_n: int = 1, norm_weight: torch.Tensor | None = None, norm_eps: float = 0.0, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: hc_mult = residual.shape[-2] hidden_size = residual.shape[-1] outer_shape = residual.shape[:-2] post_mix = torch.empty( *outer_shape, hc_mult, 1, dtype=torch.float32, device=residual.device, ) comb_mix = torch.empty( *outer_shape, hc_mult, hc_mult, dtype=torch.float32, device=residual.device, ) layer_input = torch.empty( *outer_shape, hidden_size, dtype=torch.bfloat16, device=residual.device, ) return torch.empty_like(residual), post_mix, comb_mix, layer_input direct_register_custom_op( op_name="mhc_pre_aiter", op_func=mhc_pre_aiter, mutates_args=[], fake_impl=_mhc_pre_aiter_fake, ) direct_register_custom_op( op_name="mhc_pre_delayed_aiter", op_func=mhc_pre_delayed_aiter, mutates_args=["residual_out"], fake_impl=_mhc_pre_delayed_aiter_fake, ) direct_register_custom_op( op_name="mhc_fused_post_pre_delayed_rms_norm_aiter", op_func=mhc_fused_post_pre_delayed_rms_norm_aiter, mutates_args=["residual_out"], fake_impl=_mhc_fused_post_pre_delayed_rms_norm_aiter_fake, ) direct_register_custom_op( op_name="mhc_post_aiter", op_func=mhc_post_aiter, mutates_args=[], fake_impl=_mhc_post_aiter_fake, ) direct_register_custom_op( op_name="mhc_fused_post_pre_aiter", op_func=mhc_fused_post_pre_aiter, mutates_args=[], fake_impl=_mhc_fused_post_pre_aiter_fake, )