From 46d2b23ac5047a813ebb082122166e4ae09b5f39 Mon Sep 17 00:00:00 2001 From: Fangzhou Ai <31551580+Fangzhou-Ai@users.noreply.github.com> Date: Fri, 11 Sep 2026 20:35:48 -0400 Subject: [PATCH] [ROCm][DSV4.1][Perf] Use AITER mHC for the delayed pre block (#56503) Signed-off-by: Fangzhou Ai Co-authored-by: Fangzhou Ai --- tests/kernels/test_mhc_kernels.py | 95 ++++++++++++ vllm/_aiter_ops.py | 131 +++++++++++++++++ vllm/model_executor/kernels/mhc/__init__.py | 2 + vllm/model_executor/kernels/mhc/aiter.py | 90 ++++++++++++ vllm/model_executor/kernels/mhc/triton.py | 131 +++++++++++++++++ vllm/model_executor/layers/mhc.py | 155 ++++++++++++++++++++ vllm/models/deepseek_v4_1/amd/model.py | 35 ++--- 7 files changed, 620 insertions(+), 19 deletions(-) diff --git a/tests/kernels/test_mhc_kernels.py b/tests/kernels/test_mhc_kernels.py index 61e0449cae..ab5efb6578 100644 --- a/tests/kernels/test_mhc_kernels.py +++ b/tests/kernels/test_mhc_kernels.py @@ -23,6 +23,7 @@ from vllm.model_executor.layers.mhc import ( HAS_AITER_MHC_PRE_NORM, HAS_TILELANG_MHC, MHCFusedPostPreOp, + MHCPreDelayedOp, MHCPreOp, ) from vllm.models.deepseek_v4.nvidia.model import ( @@ -697,6 +698,100 @@ def test_mhc_fused_rocm_fallback_applies_norm(monkeypatch): torch.testing.assert_close(out[3], expected_layer_input) +@pytest.mark.skipif( + not (current_platform.is_rocm() and HAS_AITER_MHC), + reason="AITER mHC required", +) +@pytest.mark.parametrize("num_tokens", [1, 2, 7, 128, 1024]) +@pytest.mark.parametrize("carried", [False, True]) +def test_mhc_pre_delayed_rocm_aiter(num_tokens, carried): + """AITER must reproduce the delayed reference on both seam variants. + + ``num_tokens`` spans the split-k choices AITER makes for the projection, + since the pre-mix is recovered from that unreduced output. + """ + set_random_seed(0) + hc_mult, hidden_size = 4, 5120 + residual, fn, hc_scale, hc_base, _ = _rocm_mhc_inputs( + num_tokens=num_tokens, hidden_size=hidden_size, hc_mult=hc_mult + ) + pre_mix = ( + torch.rand(num_tokens, hc_mult, dtype=torch.float32, device=DEVICE) + 0.5 + if carried + else None + ) + rms_eps = hc_pre_eps = hc_sinkhorn_eps = 1e-6 + args = ( + residual, + fn, + hc_scale, + hc_base, + rms_eps, + hc_pre_eps, + hc_sinkhorn_eps, + 1.0, + 20, + ) + + expected = mhc_pre_delayed_torch(*args, pre_mix=pre_mix) + actual = object.__new__(MHCPreDelayedOp).forward_hip(*args, pre_mix=pre_mix) + + for i in (0, 1, 3): + torch.testing.assert_close(actual[i], expected[i], atol=1e-4, rtol=1e-3) + # The collapse is the same FP32 multiply-and-sum in both paths. + torch.testing.assert_close(actual[2], expected[2], atol=0, rtol=0) + + +@pytest.mark.skipif( + not (current_platform.is_rocm() and HAS_AITER_MHC), + reason="AITER mHC required", +) +def test_mhc_pre_delayed_rocm_aiter_declines_unsupported(monkeypatch): + """The broadcast seam and a fused norm must not take the AITER path. + + Neither is expressible with AITER's pre kernels: the broadcast projects a + narrower ``x``, and ``mhc_pre_gemm_sqrsum`` folds no RMSNorm. What is + asserted here is the routing decision, which is what this gate owns; the + numerics of whichever fallback it lands on are covered by + ``test_deepseek_v41_mhc_pre_delayed``. + """ + set_random_seed(0) + hc_mult, hidden_size = 4, 5120 + residual, fn, hc_scale, hc_base, norm_weight = _rocm_mhc_inputs( + num_tokens=4, hidden_size=hidden_size, hc_mult=hc_mult + ) + args = (residual, fn, hc_scale, hc_base, 1e-6, 1e-6, 1e-6, 1.0, 20) + op = object.__new__(MHCPreDelayedOp) + + aiter_op = torch.ops.vllm.mhc_pre_delayed_aiter + took_aiter = False + + def spy(*spy_args, **spy_kwargs): + nonlocal took_aiter + took_aiter = True + return aiter_op(*spy_args, **spy_kwargs) + + monkeypatch.setattr(torch.ops.vllm, "mhc_pre_delayed_aiter", spy) + + x = residual[:, 0].contiguous() + broadcast_fn = fn.view(-1, hc_mult, hidden_size).sum(1) + broadcast_residual = x.unsqueeze(1).expand(-1, hc_mult, -1).contiguous() + expected = mhc_pre_delayed_torch(broadcast_residual, broadcast_fn, *args[2:], x=x) + actual = op.forward_hip(broadcast_residual, broadcast_fn, *args[2:], x=x) + assert not took_aiter, "the broadcast seam must not reach AITER" + for i in range(4): + torch.testing.assert_close(actual[i], expected[i], atol=1e-4, rtol=1e-3) + + op.forward_hip(*args, norm_weight=norm_weight, norm_eps=1e-6) + assert not took_aiter, "a fused norm must not reach AITER" + + # Positive control: the same inputs without a norm do take the AITER path, + # so the two declines above are the gate discriminating rather than the + # path being unavailable in this environment. + op.forward_hip(*args) + assert took_aiter + + @pytest.mark.skipif( not (current_platform.is_rocm() and HAS_AITER_MHC and HAS_AITER_MHC_PRE_NORM), reason="AITER mHC with fused RMSNorm required", diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index 2e1e8fc619..454aba175e 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -3599,6 +3599,137 @@ class rocm_aiter_ops: layer_input.view(*outer_shape, hidden_size), ) + @staticmethod + def mhc_pre_delayed( + residual: torch.Tensor, + fn: torch.Tensor, + hc_scale: torch.Tensor, + hc_base: torch.Tensor, + rms_eps: float, + hc_pre_eps: float, + hc_sinkhorn_eps: float, + hc_post_mult_value: float, + sinkhorn_repeat: int, + pre_mix: torch.Tensor | None = None, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """mHC pre using the pre-mix carried from the previous sublayer. + + Same gates as :meth:`mhc_pre`, but the stream collapse applies the + caller's ``pre_mix`` instead of the one computed here, and the one + computed here is returned for the next sublayer seam. AITER has no + single kernel for that shape, so drive its two stages directly: + ``mhc_pre_big_fuse`` still produces the post and comb gates (including + every sinkhorn iteration), while the pre gate is recovered from the + same split-k GEMM output and the collapse is done by the Triton + kernel. ``mhc_pre_big_fuse`` also writes a collapse we do not use; + that redundant store is the price of not having a native delayed + kernel, and is small next to the ~140 launches it replaces. + + Returns: + post_mix: shape (..., hc_mult, 1), dtype torch.float32 + comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32 + layer_input: shape (..., hidden_size), dtype torch.bfloat16 + next_pre_mix: shape (..., hc_mult), dtype torch.float32 + """ + from aiter.ops.mhc import ( + get_mhc_pre_splitk, + mhc_pre_big_fuse, + mhc_pre_gemm_sqrsum, + ) + + assert residual.dtype == torch.bfloat16 + assert fn.dtype == torch.float32 + assert hc_scale.dtype == torch.float32 + assert hc_base.dtype == torch.float32 + + hc_mult = residual.shape[-2] + hidden_size = residual.shape[-1] + hc_mult3 = hc_mult * 2 + hc_mult * hc_mult + hc_hidden_size = hc_mult * hidden_size + + assert fn.shape == (hc_mult3, hc_hidden_size) + assert hc_scale.shape == (3,) + assert hc_base.shape == (hc_mult3,) + + outer_shape = residual.shape[:-2] + residual_flat = residual.view(-1, hc_mult, hidden_size) + num_tokens = residual_flat.shape[0] + device = residual_flat.device + + if num_tokens == 0: + return ( + torch.empty(0, hc_mult, 1, dtype=torch.float32, device=device), + torch.empty(0, hc_mult, hc_mult, dtype=torch.float32, device=device), + torch.empty(0, hidden_size, dtype=torch.bfloat16, device=device), + torch.empty(0, hc_mult, dtype=torch.float32, device=device), + ) + + pre_mix_flat = None if pre_mix is None else pre_mix.view(-1, hc_mult) + + # AITER's Python wrappers allocate without explicit device arguments. + with torch.device(device): + splitk, tile_k = get_mhc_pre_splitk(num_tokens, hc_hidden_size) + # AITER pads the GEMM output to a multiple of 32 columns. + gemm_pad = torch.empty( + splitk, + num_tokens, + (hc_mult3 + 31) // 32 * 32, + dtype=torch.float32, + device=device, + ) + gemm_out = gemm_pad[:, :, :hc_mult3] + sqrsum = torch.empty(splitk, num_tokens, dtype=torch.float32, device=device) + mhc_pre_gemm_sqrsum(gemm_out, sqrsum, residual_flat, fn, tile_k, 0) + + post_mix = torch.empty( + num_tokens, hc_mult, 1, dtype=torch.float32, device=device + ) + comb_mix = torch.empty( + num_tokens, hc_mult, hc_mult, dtype=torch.float32, device=device + ) + unused_collapse = torch.empty( + num_tokens, hidden_size, dtype=torch.bfloat16, device=device + ) + mhc_pre_big_fuse( + post_mix, + comb_mix, + unused_collapse, + gemm_out, + sqrsum, + hc_scale, + hc_base, + residual_flat, + rms_eps, + hc_pre_eps, + hc_sinkhorn_eps, + hc_post_mult_value, + sinkhorn_repeat, + ) + + next_pre_mix = torch.ops.vllm.mhc_pre_mix_triton( + gemm_out, + sqrsum, + hc_scale, + hc_base, + hc_mult, + hc_hidden_size, + rms_eps, + hc_pre_eps, + ) + + if pre_mix_flat is None: + # Model entry selects residual stream zero. + layer_input = residual_flat[:, 0] + else: + layer_input = torch.ops.vllm.hc_collapse_triton(residual_flat, pre_mix_flat) + + return ( + post_mix.view(*outer_shape, hc_mult, 1), + comb_mix.view(*outer_shape, hc_mult, hc_mult), + layer_input.view(*outer_shape, hidden_size), + next_pre_mix.view(*outer_shape, hc_mult), + ) + @staticmethod def hc_head( hs_flat: torch.Tensor, diff --git a/vllm/model_executor/kernels/mhc/__init__.py b/vllm/model_executor/kernels/mhc/__init__.py index 324e3cec64..f9df7f6f2c 100644 --- a/vllm/model_executor/kernels/mhc/__init__.py +++ b/vllm/model_executor/kernels/mhc/__init__.py @@ -12,6 +12,7 @@ __all__ = [ "mhc_fused_post_pre_cuda", "hc_head_fused_kernel_cuda", "mhc_pre_aiter", + "mhc_pre_delayed_aiter", "mhc_post_aiter", "mhc_fused_post_pre_aiter", "hc_head_fused_aiter", @@ -26,6 +27,7 @@ __all__ = [ "mhc_post_torch", "mhc_fused_post_pre_torch", "hc_head_fused_torch", + "mhc_pre_mix_triton", "mhc_pre_triton", "mhc_post_triton", "mhc_fused_post_pre_triton", diff --git a/vllm/model_executor/kernels/mhc/aiter.py b/vllm/model_executor/kernels/mhc/aiter.py index ce7690ec1d..e04ce3322b 100644 --- a/vllm/model_executor/kernels/mhc/aiter.py +++ b/vllm/model_executor/kernels/mhc/aiter.py @@ -104,6 +104,90 @@ def _mhc_pre_aiter_fake( return post_mix, comb_mix, layer_input +def mhc_pre_delayed_aiter( + residual: torch.Tensor, + fn: torch.Tensor, + hc_scale: torch.Tensor, + hc_base: torch.Tensor, + rms_eps: float, + hc_pre_eps: float, + hc_sinkhorn_eps: float, + hc_post_mult_value: float, + sinkhorn_repeat: int, + pre_mix: torch.Tensor | None = None, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """mHC pre with the pre-mix carried in from the previous sublayer. + + Matches ``mhc_pre_delayed_torch``: the stream collapse uses *pre_mix* + rather than the gate computed here, and that gate is returned as the + pre-mix for the next sublayer seam. + + Args: + residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16 + fn: shape (hc_mult3, hc_mult * hidden_size), dtype torch.float32 + hc_scale: shape (3,), dtype torch.float32 + hc_base: shape (hc_mult3,), dtype torch.float32 + rms_eps: RMS normalization epsilon + hc_pre_eps: pre-mix epsilon + hc_sinkhorn_eps: sinkhorn epsilon + hc_post_mult_value: post-mix multiplier value + sinkhorn_repeat: number of sinkhorn iterations + pre_mix: shape (..., hc_mult) from the previous sublayer, or None at + model entry to select residual stream zero. + + Returns: + post_mix: shape (..., hc_mult, 1), dtype torch.float32 + comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32 + layer_input: shape (..., hidden_size), dtype torch.bfloat16 + next_pre_mix: shape (..., hc_mult), dtype torch.float32 + """ + hidden_size = residual.shape[-1] + assert hidden_size % 256 == 0 + from vllm._aiter_ops import rocm_aiter_ops + + return rocm_aiter_ops.mhc_pre_delayed( + residual, + fn, + hc_scale, + hc_base, + rms_eps, + hc_pre_eps, + hc_sinkhorn_eps, + hc_post_mult_value, + sinkhorn_repeat, + pre_mix, + ) + + +def _mhc_pre_delayed_aiter_fake( + residual: torch.Tensor, + fn: torch.Tensor, + hc_scale: torch.Tensor, + hc_base: torch.Tensor, + rms_eps: float, + hc_pre_eps: float, + hc_sinkhorn_eps: float, + hc_post_mult_value: float, + sinkhorn_repeat: int, + pre_mix: torch.Tensor | None = None, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + hc_mult = residual.shape[-2] + hidden_size = residual.shape[-1] + outer_shape = residual.shape[:-2] + return ( + torch.empty( + *outer_shape, hc_mult, 1, dtype=torch.float32, device=residual.device + ), + torch.empty( + *outer_shape, hc_mult, hc_mult, dtype=torch.float32, device=residual.device + ), + torch.empty( + *outer_shape, hidden_size, dtype=torch.bfloat16, device=residual.device + ), + torch.empty(*outer_shape, hc_mult, dtype=torch.float32, device=residual.device), + ) + + def mhc_post_aiter( x: torch.Tensor, residual: torch.Tensor, @@ -226,6 +310,12 @@ direct_register_custom_op( mutates_args=[], fake_impl=_mhc_pre_aiter_fake, ) +direct_register_custom_op( + op_name="mhc_pre_delayed_aiter", + op_func=mhc_pre_delayed_aiter, + mutates_args=[], + fake_impl=_mhc_pre_delayed_aiter_fake, +) direct_register_custom_op( op_name="mhc_post_aiter", op_func=mhc_post_aiter, diff --git a/vllm/model_executor/kernels/mhc/triton.py b/vllm/model_executor/kernels/mhc/triton.py index d94d71a6ea..8724113779 100644 --- a/vllm/model_executor/kernels/mhc/triton.py +++ b/vllm/model_executor/kernels/mhc/triton.py @@ -141,6 +141,129 @@ def _hc_collapse_triton_fake(x: Tensor, pre_mix: Tensor) -> Tensor: return torch.empty(x.shape[0], x.shape[2], dtype=x.dtype, device=x.device) +@triton.jit +def _mhc_pre_mix_kernel( + gemm_ptr, + sqrsum_ptr, + hc_scale_ptr, + hc_base_ptr, + out_ptr, + splitk, + hc_mult: tl.constexpr, + gemm_stride_k, + gemm_stride_t, + gemm_stride_j, + sqrsum_stride_k, + sqrsum_stride_t, + out_stride_t, + out_stride_j, + inv_hc_hidden, + rms_eps, + hc_pre_eps, + SPLITK_BLOCK: tl.constexpr, + HC_BLOCK: tl.constexpr, +): + """Recover the pre-mix gate from a split-k mHC pre GEMM output.""" + token_idx = tl.program_id(0).to(tl.int64) + ks = tl.arange(0, SPLITK_BLOCK) + js = tl.arange(0, HC_BLOCK) + kmask = ks < splitk + jmask = js < hc_mult + + # Only the first hc_mult GEMM columns feed the pre gate. + gemm = tl.load( + gemm_ptr + + ks[:, None] * gemm_stride_k + + token_idx * gemm_stride_t + + js[None, :] * gemm_stride_j, + mask=kmask[:, None] & jmask[None, :], + other=0.0, + ).to(tl.float32) + mixes = tl.sum(gemm, 0) + + sqrsum = tl.load( + sqrsum_ptr + ks * sqrsum_stride_k + token_idx * sqrsum_stride_t, + mask=kmask, + other=0.0, + ).to(tl.float32) + rstd = tl.rsqrt(tl.sum(sqrsum, 0) * inv_hc_hidden + rms_eps) + + scale = tl.load(hc_scale_ptr).to(tl.float32) + base = tl.load(hc_base_ptr + js, mask=jmask, other=0.0).to(tl.float32) + + pre = tl.sigmoid(mixes * rstd * scale + base) + hc_pre_eps + tl.store(out_ptr + token_idx * out_stride_t + js * out_stride_j, pre, mask=jmask) + + +def mhc_pre_mix_triton( + gemm_out: Tensor, + sqrsum: Tensor, + hc_scale: Tensor, + hc_base: Tensor, + hc_mult: int, + hc_hidden_size: int, + rms_eps: float, + hc_pre_eps: float, +) -> Tensor: + """Pre-mix gate for the delayed mHC pre, from AITER's split-k GEMM output. + + AITER's ``mhc_pre_big_fuse`` consumes the unreduced ``[splitk, tokens, + hc_mult3]`` GEMM output and the matching row square-sums, but only returns + the post and comb gates. The delayed formulation also needs the pre gate, + to carry into the next sublayer seam. It is the same slice of the same + numbers, so recover it here rather than repeating the projection. + """ + assert gemm_out.ndim == 3 and gemm_out.dtype == torch.float32 + assert sqrsum.ndim == 2 and sqrsum.dtype == torch.float32 + splitk, num_tokens = gemm_out.shape[0], gemm_out.shape[1] + assert sqrsum.shape == (splitk, num_tokens) + + out = torch.empty(num_tokens, hc_mult, dtype=torch.float32, device=gemm_out.device) + if num_tokens == 0: + return out + + _mhc_pre_mix_kernel[(num_tokens,)]( + gemm_out, + sqrsum, + hc_scale, + hc_base, + out, + splitk, + hc_mult, + gemm_out.stride(0), + gemm_out.stride(1), + gemm_out.stride(2), + sqrsum.stride(0), + sqrsum.stride(1), + out.stride(0), + out.stride(1), + 1.0 / hc_hidden_size, + rms_eps, + hc_pre_eps, + SPLITK_BLOCK=triton.next_power_of_2(splitk), + HC_BLOCK=triton.next_power_of_2(hc_mult), + num_warps=1, + # Match the separate FP32 multiply and add in the Torch reference. + enable_fp_fusion=False, + ) + return out + + +def _mhc_pre_mix_triton_fake( + gemm_out: Tensor, + sqrsum: Tensor, + hc_scale: Tensor, + hc_base: Tensor, + hc_mult: int, + hc_hidden_size: int, + rms_eps: float, + hc_pre_eps: float, +) -> Tensor: + return torch.empty( + gemm_out.shape[1], hc_mult, dtype=torch.float32, device=gemm_out.device + ) + + def hc_head_reduce_triton_kernel( x: torch.Tensor, hc_fn: torch.Tensor, @@ -216,3 +339,11 @@ direct_register_custom_op( mutates_args=[], fake_impl=_hc_collapse_triton_fake, ) + + +direct_register_custom_op( + op_name="mhc_pre_mix_triton", + op_func=mhc_pre_mix_triton, + mutates_args=[], + fake_impl=_mhc_pre_mix_triton_fake, +) diff --git a/vllm/model_executor/layers/mhc.py b/vllm/model_executor/layers/mhc.py index 6ddf4ff066..14f1018597 100644 --- a/vllm/model_executor/layers/mhc.py +++ b/vllm/model_executor/layers/mhc.py @@ -292,6 +292,161 @@ class MHCPreOp(CustomOp): ) +# --8<-- [start:mhc_pre_delayed] +@CustomOp.register("mhc_pre_delayed") +class MHCPreDelayedOp(CustomOp): + """MHC pre block using the pre-mix carried from the previous sublayer. + + Same gates as :class:`MHCPreOp`, but the stream collapse applies the + caller's ``pre_mix`` and this sublayer's pre-mix is returned for the next + sublayer seam. Returns post_mix, comb_mix, layer_input, next_pre_mix. + """ + + # --8<-- [end:mhc_pre_delayed] + @classmethod + def enabled(cls) -> bool: + return True + + def forward_cuda( + self, + residual: torch.Tensor, + fn: torch.Tensor, + hc_scale: torch.Tensor, + hc_base: torch.Tensor, + rms_eps: float, + hc_pre_eps: float, + hc_sinkhorn_eps: float, + hc_post_mult_value: float, + sinkhorn_repeat: int, + pre_mix: torch.Tensor | None = None, + x: torch.Tensor | None = None, + norm_weight: torch.Tensor | None = None, + norm_eps: float = 1e-6, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + return torch.ops.vllm.mhc_pre_delayed_tilelang( + residual, + fn, + hc_scale, + hc_base, + rms_eps, + hc_pre_eps, + hc_sinkhorn_eps, + hc_post_mult_value, + sinkhorn_repeat, + pre_mix, + x, + norm_weight, + norm_eps, + ) + + def forward_hip( + self, + residual: torch.Tensor, + fn: torch.Tensor, + hc_scale: torch.Tensor, + hc_base: torch.Tensor, + rms_eps: float, + hc_pre_eps: float, + hc_sinkhorn_eps: float, + hc_post_mult_value: float, + sinkhorn_repeat: int, + pre_mix: torch.Tensor | None = None, + x: torch.Tensor | None = None, + norm_weight: torch.Tensor | None = None, + norm_eps: float = 1e-6, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + # The AITER delayed path drives mhc_pre_gemm_sqrsum against `residual` + # and folds no RMSNorm, so it cannot serve the model-entry broadcast + # (which projects a narrower `x`) or a fused norm. Both are handled by + # TileLang, or by the reference when TileLang is unavailable. + if ( + x is None + and norm_weight is None + and _aiter_mhc_supported(residual, None, supports_norm=False) + ): + return torch.ops.vllm.mhc_pre_delayed_aiter( + residual, + fn, + hc_scale, + hc_base, + rms_eps, + hc_pre_eps, + hc_sinkhorn_eps, + hc_post_mult_value, + sinkhorn_repeat, + pre_mix, + ) + if HAS_TILELANG_MHC: + return torch.ops.vllm.mhc_pre_delayed_tilelang( + residual, + fn, + hc_scale, + hc_base, + rms_eps, + hc_pre_eps, + hc_sinkhorn_eps, + hc_post_mult_value, + sinkhorn_repeat, + pre_mix, + x, + norm_weight, + norm_eps, + ) + return self.forward_native( + residual, + fn, + hc_scale, + hc_base, + rms_eps, + hc_pre_eps, + hc_sinkhorn_eps, + hc_post_mult_value, + sinkhorn_repeat, + pre_mix, + x, + norm_weight, + norm_eps, + ) + + def forward_native( + self, + residual: torch.Tensor, + fn: torch.Tensor, + hc_scale: torch.Tensor, + hc_base: torch.Tensor, + rms_eps: float, + hc_pre_eps: float, + hc_sinkhorn_eps: float, + hc_post_mult_value: float, + sinkhorn_repeat: int, + pre_mix: torch.Tensor | None = None, + x: torch.Tensor | None = None, + norm_weight: torch.Tensor | None = None, + norm_eps: float = 1e-6, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + post_mix, comb_mix, layer_input, next_pre_mix = ( + mhc_kernels.mhc_pre_delayed_torch( + residual, + fn, + hc_scale, + hc_base, + rms_eps, + hc_pre_eps, + hc_sinkhorn_eps, + hc_post_mult_value, + sinkhorn_repeat, + pre_mix=pre_mix, + x=x, + ) + ) + return ( + post_mix, + comb_mix, + _apply_mhc_norm(layer_input, norm_weight, norm_eps), + next_pre_mix, + ) + + # --8<-- [start:mhc_post] @CustomOp.register("mhc_post") class MHCPostOp(CustomOp): diff --git a/vllm/models/deepseek_v4_1/amd/model.py b/vllm/models/deepseek_v4_1/amd/model.py index 069ff5f931..898d3ab7a0 100644 --- a/vllm/models/deepseek_v4_1/amd/model.py +++ b/vllm/models/deepseek_v4_1/amd/model.py @@ -17,15 +17,12 @@ from vllm.distributed import ( ) from vllm.forward_context import get_forward_context, is_forward_context_available from vllm.logger import init_logger -from vllm.model_executor.kernels.mhc.torch import ( - mhc_post_torch, - mhc_pre_delayed_torch, -) from vllm.model_executor.layers.fused_moe import ( fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.mhc import MHCPostOp, MHCPreDelayedOp from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, @@ -248,11 +245,13 @@ class DeepseekV4DecoderLayer(nn.Module): ), requires_grad=False, ) + self.mhc_pre_delayed = MHCPreDelayedOp() + self.mhc_post = MHCPostOp() @staticmethod def _hc_collapse(x: torch.Tensor, pre_mix: torch.Tensor) -> torch.Tensor: - """Reference hyper-connection collapse used by DSpark on ROCm.""" - return (pre_mix.unsqueeze(-1) * x.float()).sum(dim=1).to(x.dtype) + """Hyper-connection collapse used by DSpark on ROCm.""" + return torch.ops.vllm.hc_collapse_triton(x, pre_mix) def forward( self, @@ -275,7 +274,7 @@ class DeepseekV4DecoderLayer(nn.Module): # copies and the identity pre-mix selects copy 0. assert self.hc_attn_fn_broadcast is not None residual = x.unsqueeze(1).expand(-1, self.hc_mult, -1).contiguous() - post_mix, res_mix, x, attn_pre = mhc_pre_delayed_torch( + post_mix, res_mix, x, attn_pre = self.mhc_pre_delayed( residual, self.hc_attn_fn_broadcast, self.hc_attn_scale, @@ -289,7 +288,7 @@ class DeepseekV4DecoderLayer(nn.Module): ) else: residual = x - post_mix, res_mix, x, attn_pre = mhc_pre_delayed_torch( + post_mix, res_mix, x, attn_pre = self.mhc_pre_delayed( residual, self.hc_attn_fn, self.hc_attn_scale, @@ -302,7 +301,7 @@ class DeepseekV4DecoderLayer(nn.Module): pre_mix=pre_mix, ) else: - residual = mhc_post_torch(x, residual, post_mix, res_mix) + residual = self.mhc_post(x, residual, post_mix, res_mix) if self.engram is not None and engram_hashes is not None: # Engram injection happens between the previous sublayer's # post and this block's pre, on the full hc stream, so the @@ -312,7 +311,7 @@ class DeepseekV4DecoderLayer(nn.Module): engram_hashes[:, self.engram.layer_hash_index], engram_mask, ) - post_mix, res_mix, x, attn_pre = mhc_pre_delayed_torch( + post_mix, res_mix, x, attn_pre = self.mhc_pre_delayed( residual, self.hc_attn_fn, self.hc_attn_scale, @@ -333,8 +332,8 @@ class DeepseekV4DecoderLayer(nn.Module): if self.use_sequence_parallel: x = sp_reduce_scatter(x) - residual = mhc_post_torch(x, residual, post_mix, res_mix) - post_mix, res_mix, x, ffn_pre = mhc_pre_delayed_torch( + residual = self.mhc_post(x, residual, post_mix, res_mix) + post_mix, res_mix, x, ffn_pre = self.mhc_pre_delayed( residual, self.hc_ffn_fn, self.hc_ffn_scale, @@ -453,6 +452,8 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): else: self.norm = PPMissingLayer() + self.mhc_post = MHCPostOp() + spec_config = vllm_config.speculative_config needs_mtp_hidden_states = spec_config is not None and ( spec_config.use_eagle() or spec_config.uses_draft_model() @@ -603,7 +604,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): ) if idx + 1 in self.aux_hidden_state_layers: # Reconstruct the aux hidden state for draft models - aux_recon = mhc_post_torch(hidden_states, residual, post_mix, res_mix) + aux_recon = self.mhc_post(hidden_states, residual, post_mix, res_mix) aux_hidden_state = aux_recon.mean(dim=1) if self.use_sequence_parallel: aux_hidden_state = sp_all_gather(aux_hidden_state)[:full_num_tokens] @@ -614,7 +615,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): if self.end_layer in self.aux_hidden_state_layers: hidden_states = final_aux_recon else: - hidden_states = mhc_post_torch( + hidden_states = self.mhc_post( hidden_states, residual, post_mix, res_mix ) @@ -635,11 +636,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): # mixes — the mix the reference applies via # ``last_layer.hc_pre(h, pre_mix)`` (v4.1 has no learned hc_head). assert pre_mix is not None - hidden_states = ( - (pre_mix.unsqueeze(-1) * hidden_states.float()) - .sum(dim=1) - .to(hidden_states.dtype) - ) + hidden_states = torch.ops.vllm.hc_collapse_triton(hidden_states, pre_mix) hidden_states = self.norm(hidden_states) if len(aux_hidden_states) > 0: return hidden_states, aux_hidden_states