diff --git a/tests/kernels/test_mhc_kernels.py b/tests/kernels/test_mhc_kernels.py new file mode 100644 index 0000000000..24d9038f48 --- /dev/null +++ b/tests/kernels/test_mhc_kernels.py @@ -0,0 +1,142 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import pytest +import torch + +import vllm.model_executor.layers.mhc as mhc_ops # noqa: F401 +from vllm.platforms import current_platform +from vllm.utils.torch_utils import set_random_seed + +DEVICE = current_platform.device_type + + +def sinkhorn_normalize_ref(x: torch.Tensor, repeat: int, eps: float) -> torch.Tensor: + x = x.softmax(-1) + eps + x = x / (x.sum(-2, keepdim=True) + eps) + for _ in range(repeat - 1): + x = x / (x.sum(-1, keepdim=True) + eps) + x = x / (x.sum(-2, keepdim=True) + eps) + return x + + +def mhc_pre_ref( + residual: torch.Tensor, + fn: torch.Tensor, + hc_scale: torch.Tensor, + hc_base: torch.Tensor, + rms_eps: float, + hc_pre_eps: float, + hc_sinkhorn_eps: float, + hc_post_mult_value: float, + sinkhorn_repeat: int, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """mHC pre reference kernel from tilelang repo: https://github.com/tile-ai/tilelang/blob/d135bd1cd2d2eee74fbb41dd0a0831a427194c86/examples/deepseek_mhc/example_mhc_pre.py#L303""" + hc_mult = residual.shape[-2] + + residual_flat = residual.flatten(-2, -1).float() + sqrsum = residual_flat.square().sum(-1) + mixes = ( + residual_flat @ fn.T * (sqrsum.unsqueeze(-1) / fn.shape[-1] + rms_eps).rsqrt() + ) + + hc_scale = torch.cat( + [ + hc_scale[0].expand(hc_mult), + hc_scale[1].expand(hc_mult), + hc_scale[2].expand(hc_mult * hc_mult), + ], + ) + mixes = mixes * hc_scale + hc_base + + pre_mix = mixes[:, :hc_mult].sigmoid().unsqueeze(-1) + hc_pre_eps + post_mix = ( + mixes[:, hc_mult : 2 * hc_mult].sigmoid() * hc_post_mult_value + ).unsqueeze(-1) + res_mix = mixes[:, 2 * hc_mult :].view(-1, hc_mult, hc_mult) + + res_mix = sinkhorn_normalize_ref( + res_mix, repeat=sinkhorn_repeat, eps=hc_sinkhorn_eps + ) + + layer_input = (residual * pre_mix).sum(-2).bfloat16() + + return post_mix, res_mix, layer_input + + +def mhc_post_ref( + x: torch.Tensor, + residual: torch.Tensor, + post_layer_mix: torch.Tensor, + comb_res_mix: torch.Tensor, +) -> torch.Tensor: + """mHC post reference kernel from tilelang repo: https://github.com/tile-ai/tilelang/blob/d135bd1cd2d2eee74fbb41dd0a0831a427194c86/examples/deepseek_mhc/example_mhc_post.py#L68""" + term2 = torch.bmm(comb_res_mix.mT, residual.float()) + return (x.float().unsqueeze(-2) * post_layer_mix + term2).bfloat16() + + +@pytest.mark.skipif( + not current_platform.is_cuda(), + reason="CUDA required", +) +@pytest.mark.parametrize("num_tokens", [1, 4, 8, 128]) +@pytest.mark.parametrize("hidden_size", [4096, 7168]) +@pytest.mark.parametrize("hc_mult", [4]) +def test_mhc_fused_post_pre(num_tokens, hidden_size, hc_mult): + torch.set_default_device(DEVICE) + set_random_seed(0) + + x = torch.randn((num_tokens, hidden_size), dtype=torch.bfloat16) + residual = torch.randn((num_tokens, hc_mult, hidden_size), dtype=torch.bfloat16) + post_layer_mix = torch.randn((num_tokens, hc_mult, 1), dtype=torch.float32) + comb_res_mix = torch.randn((num_tokens, hc_mult, hc_mult), dtype=torch.float32) + + hc_mult2 = hc_mult * hc_mult + hc_mult3 = hc_mult * 2 + hc_mult2 + fn = ( + torch.randn((hc_mult3, hc_mult, hidden_size), dtype=torch.float) + * 1e-4 + * (1 + torch.arange(hc_mult).mul(0.01).view(1, -1, 1)) + ).flatten(1, 2) + hc_scale = torch.randn((3,), dtype=torch.float) * 0.1 + hc_base = torch.randn((hc_mult3,), dtype=torch.float) * 0.1 + + hc_sinkhorn_eps = hc_pre_eps = rms_eps = 1e-6 + sinkhorn_repeat = 20 + hc_post_alpha = 1.0 + + def run_ref(): + residual_ref = mhc_post_ref(x, residual, post_layer_mix, comb_res_mix) + post_mix_ref, res_mix_ref, layer_input_ref = mhc_pre_ref( + residual_ref, + fn, + hc_scale, + hc_base, + rms_eps, + hc_pre_eps, + hc_sinkhorn_eps, + hc_post_alpha, + sinkhorn_repeat, + ) + return residual_ref, post_mix_ref, res_mix_ref, layer_input_ref + + residual_ref, post_mix_ref, res_mix_ref, layer_input_ref = run_ref() + + residual, post_mix, res_mix, x = torch.ops.vllm.mhc_fused_post_pre( + x, + residual, + post_layer_mix, + comb_res_mix, + fn, + hc_scale, + hc_base, + rms_eps, + hc_pre_eps, + hc_sinkhorn_eps, + hc_post_alpha, + sinkhorn_repeat, + ) + + torch.testing.assert_close(residual, residual_ref, atol=1e-2, rtol=1e-2) + torch.testing.assert_close(post_mix, post_mix_ref, atol=1e-2, rtol=1e-2) + torch.testing.assert_close(res_mix, res_mix_ref, atol=1e-2, rtol=1e-2) + torch.testing.assert_close(x, layer_input_ref, atol=1e-2, rtol=1e-2) diff --git a/vllm/model_executor/layers/mhc.py b/vllm/model_executor/layers/mhc.py index cbc5ec2962..22193a12b7 100644 --- a/vllm/model_executor/layers/mhc.py +++ b/vllm/model_executor/layers/mhc.py @@ -441,6 +441,131 @@ def mhc_post_tilelang( T.pdl_trigger() +@tilelang.jit( + pass_configs={ + tilelang.PassConfigKey.TL_DISABLE_WARP_SPECIALIZED: True, + tilelang.PassConfigKey.TL_DISABLE_TMA_LOWER: True, + tilelang.PassConfigKey.TL_PTXAS_REGISTER_USAGE_LEVEL: 10, + }, +) +def mhc_fused_tilelang( + comb_mix, + residual_in, + post_mix, + x_in, + weight_t, + yp_out, + rp_out, + residual_out, + hc: int, + hidden: int, + n_out: int, + n_thr: int = 256, + h_blk: int = 256, + tile_n: int = 1, + split_k: int = 1, +) -> tilelang.JITKernel: + """Fused mhc post-mapping + pre-norm GEMM FMA""" + m = T.dynamic("num_tokens") + split_k = T.dynamic("split_k") + h = hidden + h_blk = math.gcd(hidden, h_blk) + h_per_split = h // split_k + n_tiles = n_out // tile_n + + comb_mix: T.Tensor((m, hc, hc), T.float32) # type: ignore[no-redef, valid-type] + residual_in: T.Tensor((m, hc, h), T.bfloat16) # type: ignore[no-redef, valid-type] + post_mix: T.Tensor((m, hc), T.float32) # type: ignore[no-redef, valid-type] + x_in: T.Tensor((m, h), T.bfloat16) # type: ignore[no-redef, valid-type] + weight_t: T.Tensor((n_out, hc, h), T.float32) # type: ignore[no-redef, valid-type] + yp_out: T.Tensor((split_k, m, n_out), T.float32) # type: ignore[no-redef, valid-type] + rp_out: T.Tensor((split_k, m), T.float32) # type: ignore[no-redef, valid-type] + residual_out: T.Tensor((m, hc, h), T.bfloat16) # type: ignore[no-redef, valid-type] + + h_iters = h_per_split // n_thr + num_warps = n_thr // 32 + + with T.Kernel(m, n_tiles, split_k, threads=n_thr) as (i_n, i_nt, i_ks): + tid = T.get_thread_binding() + warp_id = T.get_warp_idx() + lane = T.get_lane_idx() + + s_warp = T.alloc_shared((num_warps, tile_n + 1), T.float32) + s_post = T.alloc_shared((hc,), T.float32) + s_comb = T.alloc_shared((hc, hc), T.float32) + + pm = T.alloc_local((hc,), T.float32) + cm = T.alloc_local((hc, hc), T.float32) + acc = T.alloc_local((tile_n,), T.float32) + sqr = T.alloc_local((1,), T.float32) + new_r = T.alloc_local((hc,), T.float32) + + T.clear(acc) + T.clear(sqr) + h_split_start = i_ks * h_per_split + + T.pdl_sync() + + T.copy(post_mix[i_n, 0], s_post) + T.copy(comb_mix[i_n, 0, 0], s_comb) + + for j in T.unroll(hc): + pm[j] = s_post[j] + for j in T.unroll(hc): + for k in T.unroll(hc): + cm[k, j] = s_comb[k, j] + + # Each thread owns h_iters elements of the k-split's h slice. + for it in T.serial(h_iters): + h_idx = h_split_start + it * n_thr + tid + + # Compute new residual from layer output and past residual + for j in T.unroll(hc): + new_r[j] = pm[j] * x_in[i_n, h_idx] + for k in T.unroll(hc): + new_r[j] += cm[k, j] * residual_in[i_n, k, h_idx] + + # populate residual_out and compute sqr sum + if i_nt == 0: + for j in T.unroll(hc): + residual_out[i_n, j, h_idx] = new_r[j] + sqr[0] += new_r[j] * new_r[j] + + # Per-thread FMA into acc[n] + for n in T.unroll(tile_n): + for j in T.unroll(hc): + acc[n] += weight_t[i_nt * tile_n + n, j, h_idx] * new_r[j] + + for n in T.unroll(tile_n): + acc[n] = T.warp_reduce_sum(acc[n]) + if i_nt == 0: + sqr[0] = T.warp_reduce_sum(sqr[0]) + + # Cross-warp reduce via shared mem + if lane == 0: + for n in T.unroll(tile_n): + s_warp[warp_id, n] = acc[n] + if i_nt == 0: + s_warp[warp_id, tile_n] = sqr[0] + T.sync_threads() + + # Warp 0 does the final cross-warp sum and writes outputs + if warp_id == 0: + if lane < tile_n: + v = T.alloc_var(T.float32, init=0.0) + for w in T.unroll(num_warps): + v += s_warp[w, lane] + yp_out[i_ks, i_n, i_nt * tile_n + lane] = v + + if i_nt == 0 and lane == 0: + v2 = T.alloc_var(T.float32, init=0.0) + for w in T.unroll(num_warps): + v2 += s_warp[w, tile_n] + rp_out[i_ks, i_n] = v2 + + T.pdl_trigger() + + def mhc_post( x: torch.Tensor, residual: torch.Tensor, @@ -468,6 +593,218 @@ def mhc_post( return out +def mhc_fused_post_pre( + x: torch.Tensor, + residual: torch.Tensor, + post_layer_mix: torch.Tensor, + comb_res_mix: torch.Tensor, + fn: torch.Tensor, + hc_scale: torch.Tensor, + hc_base: torch.Tensor, + rms_eps: float, + hc_pre_eps: float, + hc_sinkhorn_eps: float, + hc_post_mult_value: float, + sinkhorn_repeat: int, + n_splits: int = 1, + tile_n: int = 1, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """ + Run one MHC post block followed by the next MHC pre block. + + Returns: + residual_cur: post-mapped residual, shape (..., hc_mult, hidden_size) + post_mix_cur: shape (..., hc_mult, 1) + comb_mix_cur: shape (..., hc_mult, hc_mult) + layer_input_cur: shape (..., hidden_size) + """ + + assert residual.dtype == torch.bfloat16 + assert x.dtype == torch.bfloat16 + assert post_layer_mix.dtype == torch.float32 + assert comb_res_mix.dtype == torch.float32 + assert fn.dtype == torch.float32 + assert hc_scale.dtype == torch.float32 + assert hc_base.dtype == torch.float32 + + hc_mult = residual.shape[-2] + hidden_size = residual.shape[-1] + hc_mult2 = hc_mult * hc_mult + hc_mult3 = hc_mult * 2 + hc_mult2 + hc_hidden_size = hc_mult * hidden_size + outer_shape = residual.shape[:-2] + + assert x.shape == (*outer_shape, hidden_size) + assert post_layer_mix.shape in ( + (*outer_shape, hc_mult, 1), + (*outer_shape, hc_mult), + ) + assert comb_res_mix.shape == (*outer_shape, hc_mult, hc_mult) + assert fn.shape == (hc_mult3, hc_hidden_size) + assert hc_scale.shape == (3,) + assert hc_base.shape == (hc_mult3,) + + assert n_splits in (1, 2, 4, 8) + assert hidden_size % n_splits == 0 + + residual_flat = residual.view(-1, hc_mult, hidden_size) + num_tokens = residual_flat.shape[0] + x_flat = x.view(num_tokens, hidden_size) + post_layer_mix_flat = post_layer_mix.view(num_tokens, hc_mult) + comb_res_mix_flat = comb_res_mix.view(num_tokens, hc_mult, hc_mult) + + fma_token_threshold = 16 + if num_tokens <= fma_token_threshold: + # TODO(gnovack): investigate autotuning these heuristics + tile_n = 2 if num_tokens < 8 else 3 + n_splits = 8 if (num_tokens < 8 and hidden_size <= 4096) else 4 + else: + # these number are from deepgemm kernel impl + block_k = 64 + block_m = 64 + n_splits = compute_num_split(block_k, hc_hidden_size, cdiv(num_tokens, block_m)) + + gemm_out_mul = torch.empty( + n_splits, + num_tokens, + hc_mult3, + dtype=torch.float32, + device=residual.device, + ) + gemm_out_sqrsum = torch.empty( + n_splits, + num_tokens, + dtype=torch.float32, + device=residual.device, + ) + residual_cur = torch.empty_like(residual_flat) + post_mix_cur = torch.empty( + num_tokens, + hc_mult, + dtype=torch.float32, + device=residual.device, + ) + comb_mix_cur = torch.empty( + num_tokens, + hc_mult2, + dtype=torch.float32, + device=residual.device, + ) + layer_input_cur = torch.empty( + num_tokens, + hidden_size, + dtype=torch.bfloat16, + device=residual.device, + ) + + if num_tokens <= fma_token_threshold: + mhc_fused_tilelang( + comb_res_mix_flat, + residual_flat, + post_layer_mix_flat, + x_flat, + fn.view(hc_mult3, hc_mult, hidden_size), + gemm_out_mul, + gemm_out_sqrsum, + residual_cur, + hc_mult, + hidden_size, + hc_mult3, + tile_n=tile_n, + n_splits=n_splits, + ) + else: + mhc_post_tilelang( + comb_res_mix_flat, + residual_flat, + post_layer_mix_flat, + x_flat, + residual_cur, + residual.shape[-2], + residual.shape[-1], + ) + + from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm + + tf32_hc_prenorm_gemm( + residual_cur.view(num_tokens, hc_mult * hidden_size), + fn, + gemm_out_mul, + gemm_out_sqrsum, + n_splits, + ) + + mhc_pre_big_fuse_tilelang( + gemm_out_mul, + gemm_out_sqrsum, + hc_scale, + hc_base, + residual_cur, + post_mix_cur, + comb_mix_cur, + layer_input_cur, + hidden_size, + rms_eps, + hc_pre_eps, + hc_sinkhorn_eps, + hc_post_mult_value, + sinkhorn_repeat, + n_splits, + hc_mult, + ) + + return ( + residual_cur.view(*outer_shape, hc_mult, hidden_size), + post_mix_cur.view(*outer_shape, hc_mult, 1), + comb_mix_cur.view(*outer_shape, hc_mult, hc_mult), + layer_input_cur.view(*outer_shape, hidden_size), + ) + + +def _mhc_fused_post_pre_fake( + x: torch.Tensor, + residual: torch.Tensor, + post_layer_mix: torch.Tensor, + comb_res_mix: torch.Tensor, + fn: torch.Tensor, + hc_scale: torch.Tensor, + hc_base: torch.Tensor, + rms_eps: float, + hc_pre_eps: float, + hc_sinkhorn_eps: float, + hc_post_mult_value: float, + sinkhorn_repeat: int, + n_splits: int = 1, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + hc_mult = residual.shape[-2] + hidden_size = residual.shape[-1] + outer_shape = residual.shape[:-2] + + residual_cur = torch.empty_like(residual) + post_mix_cur = torch.empty( + *outer_shape, + hc_mult, + 1, + dtype=torch.float32, + device=residual.device, + ) + comb_mix_cur = torch.empty( + *outer_shape, + hc_mult, + hc_mult, + dtype=torch.float32, + device=residual.device, + ) + layer_input_cur = torch.empty( + *outer_shape, + hidden_size, + dtype=torch.bfloat16, + device=residual.device, + ) + + return residual_cur, post_mix_cur, comb_mix_cur, layer_input_cur + + def _mhc_post_fake( x: torch.Tensor, residual: torch.Tensor, @@ -489,6 +826,12 @@ direct_register_custom_op( mutates_args=[], fake_impl=_mhc_post_fake, ) +direct_register_custom_op( + op_name="mhc_fused_post_pre", + op_func=mhc_fused_post_pre, + mutates_args=[], + fake_impl=_mhc_fused_post_pre_fake, +) @tilelang.jit( diff --git a/vllm/model_executor/models/deepseek_v4.py b/vllm/model_executor/models/deepseek_v4.py index 8c1fc56c0e..791d9b1bf5 100644 --- a/vllm/model_executor/models/deepseek_v4.py +++ b/vllm/model_executor/models/deepseek_v4.py @@ -1203,23 +1203,53 @@ class DeepseekV4DecoderLayer(nn.Module): x: torch.Tensor, positions: torch.Tensor, input_ids: torch.Tensor | None, + post_mix: torch.Tensor | None, + res_mix: torch.Tensor | None, + residual: torch.Tensor | None, ) -> torch.Tensor: - residual = x - x, post, comb = self.hc_pre( - x, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base - ) + if residual is None: + # Run standalone hc_pre on first layer + residual = x + x, post_mix, res_mix = self.hc_pre( + x, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base + ) + else: + residual, post_mix, res_mix, x = torch.ops.vllm.mhc_fused_post_pre( + x, + residual, + post_mix, + res_mix, + self.hc_attn_fn, + self.hc_attn_scale, + self.hc_attn_base, + self.rms_norm_eps, + self.hc_eps, + self.hc_eps, + self.hc_post_alpha, + self.hc_sinkhorn_iters, + ) + x = self.attn_norm(x) x = self.attn(positions, x, None) - x = self.hc_post(x, residual, post, comb) - residual = x - x, post, comb = self.hc_pre( - x, self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base + residual, post_mix, res_mix, x = torch.ops.vllm.mhc_fused_post_pre( + x, + residual, + post_mix, + res_mix, + self.hc_ffn_fn, + self.hc_ffn_scale, + self.hc_ffn_base, + self.rms_norm_eps, + self.hc_eps, + self.hc_eps, + self.hc_post_alpha, + self.hc_sinkhorn_iters, ) + x = self.ffn_norm(x) x = self.ffn(x, input_ids) - x = self.hc_post(x, residual, post, comb) - return x + return x, residual, post_mix, res_mix @support_torch_compile @@ -1369,12 +1399,19 @@ class DeepseekV4Model(nn.Module): if self.use_mega_moe: input_ids = input_ids.to(torch.int64) + + residual, post_mix, res_mix = None, None, None for layer in islice(self.layers, self.start_layer, self.end_layer): - hidden_states = layer( + hidden_states, residual, post_mix, res_mix = layer( hidden_states, positions, input_ids, + post_mix, + res_mix, + residual, ) + else: + hidden_states = layer.hc_post(hidden_states, residual, post_mix, res_mix) if not get_pp_group().is_last_rank: return IntermediateTensors({"hidden_states": hidden_states})