From 7ac2ec758208a27bbe0c4155a136969463e8e6c8 Mon Sep 17 00:00:00 2001 From: yinfengLiu Date: Wed, 5 Aug 2026 00:32:13 +0800 Subject: [PATCH] [Kimi-K3][AMD] Fuse AttnRes state updates and norms (#50593) Signed-off-by: Liuyinfeng01 <199041580+LiuYinfeng01@users.noreply.github.com> Co-authored-by: Liuyinfeng01 <199041580+LiuYinfeng01@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- tests/models/kimi_k3/test_amd_attn_res.py | 91 ++++++++++++++ vllm/models/kimi_k3/amd/linear.py | 25 ++-- vllm/models/kimi_k3/amd/ops/attn_res.py | 143 +++++++++++++++------- 3 files changed, 209 insertions(+), 50 deletions(-) diff --git a/tests/models/kimi_k3/test_amd_attn_res.py b/tests/models/kimi_k3/test_amd_attn_res.py index f6dfaa422b..80d5e1936f 100644 --- a/tests/models/kimi_k3/test_amd_attn_res.py +++ b/tests/models/kimi_k3/test_amd_attn_res.py @@ -88,11 +88,15 @@ def test_amd_attn_res_matches_reference( actual = attn_res( prefix, + None, blocks, norm_weight, qk_weight, + None, num_blocks, + -1, eps, + 0.0, ) torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2) @@ -100,3 +104,90 @@ def test_amd_attn_res_matches_reference( torch.testing.assert_close(blocks, original_blocks, atol=0, rtol=0) assert actual.shape == prefix.shape assert actual.is_contiguous() + + +@pytest.mark.parametrize( + ( + "num_tokens", + "num_blocks", + "hidden_size", + "has_delta", + "write_block", + "apply_output_norm", + ), + [ + pytest.param(1, 0, 128, False, True, True, id="empty-write-norm"), + pytest.param(7, 1, 1024, True, False, True, id="single-add-norm"), + pytest.param(17, 5, 7168, True, True, True, id="padded-write-add"), + pytest.param(3, 8, 7168, True, False, True, id="full-add-norm"), + pytest.param(320, 4, 7168, True, False, False, id="prefill-add"), + ], +) +def test_amd_attn_res_fused_contract( + num_tokens: int, + num_blocks: int, + hidden_size: int, + has_delta: bool, + write_block: bool, + apply_output_norm: bool, +) -> None: + torch.manual_seed(42) + eps = 1e-5 + output_eps = 2e-5 + block_capacity = 9 + prefix = _randn_with_row_padding(num_tokens, hidden_size, padding=7) + delta = ( + _randn_with_row_padding(num_tokens, hidden_size, padding=11) + if has_delta + else None + ) + blocks = _randn_with_row_padding( + num_tokens, block_capacity, hidden_size, padding=13 + ) + norm_weight = 1 + 0.1 * torch.randn( + hidden_size, device="cuda", dtype=torch.bfloat16 + ) + qk_weight = ( + torch.randn(hidden_size, device="cuda", dtype=torch.bfloat16) / hidden_size**0.5 + ) + output_norm_weight = ( + 1 + 0.1 * torch.randn(hidden_size, device="cuda", dtype=torch.bfloat16) + if apply_output_norm + else None + ) + expected_prefix = prefix.clone() + if delta is not None: + expected_prefix = expected_prefix + delta + values = torch.cat( + (blocks[:, :num_blocks].clone(), expected_prefix.unsqueeze(1)), dim=1 + ) + keys = F.rms_norm(values.float(), (hidden_size,), norm_weight.float(), eps) + probs = (keys @ qk_weight.float()).softmax(dim=-1) + expected = torch.matmul(probs.unsqueeze(1), values.float()).squeeze(1) + if output_norm_weight is not None: + expected = F.rms_norm( + expected, (hidden_size,), output_norm_weight.float(), output_eps + ) + expected = expected.to(prefix.dtype) + original_blocks = blocks.clone() + block_write_idx = num_blocks if write_block else -1 + + actual = attn_res( + prefix, + delta, + blocks, + norm_weight, + qk_weight, + output_norm_weight, + num_blocks, + block_write_idx, + eps, + output_eps, + ) + + torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2) + torch.testing.assert_close(prefix, expected_prefix, atol=0, rtol=0) + if write_block: + original_blocks[:, block_write_idx].copy_(expected_prefix) + torch.testing.assert_close(blocks, original_blocks, atol=0, rtol=0) + assert actual.is_contiguous() diff --git a/vllm/models/kimi_k3/amd/linear.py b/vllm/models/kimi_k3/amd/linear.py index 181704ebf9..54f3aae63d 100644 --- a/vllm/models/kimi_k3/amd/linear.py +++ b/vllm/models/kimi_k3/amd/linear.py @@ -141,17 +141,22 @@ def _apply_attn_res( proj: ReplicatedLinear, norm: RMSNorm, num_valid_blocks: int, + *, + delta: torch.Tensor | None = None, + output_norm: RMSNorm | None = None, + block_write_idx: int = -1, ) -> torch.Tensor: - if num_valid_blocks <= 0: - return prefix_sum - return attn_res( prefix_sum, + delta, block_residual, norm.weight, proj.weight.squeeze(0), + None if output_norm is None else output_norm.weight, num_valid_blocks, + block_write_idx, norm.variance_epsilon, + 0.0 if output_norm is None else output_norm.variance_epsilon, ) @@ -610,19 +615,20 @@ class KimiDecoderLayer(nn.Module): self.self_attention_res_proj, self.self_attention_res_norm, self.prev_valid_blocks, + output_norm=self.input_layernorm, + block_write_idx=(self.block_write_idx if self.is_block_write_layer else -1), ) if self.is_block_write_layer: - block_residual[:, self.block_write_idx, :].copy_(prefix_sum) prefix_sum = None - hidden_states = self.input_layernorm(hidden_states) hidden_states = self._run_self_attn(positions, hidden_states) - if prefix_sum is not None: - prefix_sum = prefix_sum + hidden_states - else: + if prefix_sum is None: prefix_sum = hidden_states + prefix_delta = None + else: + prefix_delta = hidden_states mlp_valid_blocks = self.prev_valid_blocks + ( 1 if self.is_block_write_layer else 0 @@ -633,9 +639,10 @@ class KimiDecoderLayer(nn.Module): self.mlp_res_proj, self.mlp_res_norm, mlp_valid_blocks, + delta=prefix_delta, + output_norm=self.post_attention_layernorm, ) - hidden_states = self.post_attention_layernorm(hidden_states) hidden_states = self.mlp(hidden_states) prefix_sum = prefix_sum + hidden_states return prefix_sum, block_residual diff --git a/vllm/models/kimi_k3/amd/ops/attn_res.py b/vllm/models/kimi_k3/amd/ops/attn_res.py index c00a3422ca..5802e60367 100644 --- a/vllm/models/kimi_k3/amd/ops/attn_res.py +++ b/vllm/models/kimi_k3/amd/ops/attn_res.py @@ -7,6 +7,7 @@ # the following copyright notice: # Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li + import torch from vllm.triton_utils import tl, triton @@ -15,17 +16,25 @@ from vllm.triton_utils import tl, triton @triton.jit def _attn_res_kernel( prefix_ptr, + delta_ptr, blocks_ptr, norm_weight_ptr, qk_weight_ptr, + output_norm_weight_ptr, output_ptr, stride_prefix_m: tl.constexpr, + stride_delta_m: tl.constexpr, stride_block_m: tl.constexpr, stride_block_r: tl.constexpr, stride_output_m: tl.constexpr, num_blocks: tl.constexpr, hidden_size: tl.constexpr, + block_write_idx: tl.constexpr, eps: tl.constexpr, + output_norm_eps: tl.constexpr, + HAS_DELTA: tl.constexpr, + WRITE_BLOCK: tl.constexpr, + APPLY_OUTPUT_NORM: tl.constexpr, BLOCK_L: tl.constexpr, BLOCK_D: tl.constexpr, ): @@ -33,51 +42,93 @@ def _attn_res_kernel( d_offsets = tl.max_contiguous(tl.arange(0, BLOCK_D), BLOCK_D) d_mask = d_offsets < hidden_size - prefix = tl.load( + updated_prefix = tl.load( prefix_ptr + row_idx * stride_prefix_m + d_offsets, mask=d_mask, other=0.0, ).to(tl.float32) - input_qk_weight = tl.load(norm_weight_ptr + d_offsets, mask=d_mask, other=0.0).to( - tl.float32 - ) * tl.load(qk_weight_ptr + d_offsets, mask=d_mask, other=0.0).to(tl.float32) - - max_logit = tl.full((), -float("inf"), tl.float32) - denominator = tl.zeros((), tl.float32) - mixed = tl.zeros((BLOCK_D,), tl.float32) - num_sources = num_blocks + 1 - - for source_tile in range(tl.cdiv(num_sources, BLOCK_L)): - source_offsets = source_tile * BLOCK_L + tl.arange(0, BLOCK_L) - source_mask = source_offsets < num_sources - is_prefix = source_offsets == num_blocks - block_ptrs = ( + if HAS_DELTA: + delta = tl.load( + delta_ptr + row_idx * stride_delta_m + d_offsets, + mask=d_mask, + other=0.0, + ).to(tl.float32) + updated_prefix += delta + # Match the BF16 prefix-add result before using it as a residual source. + updated_prefix = updated_prefix.to(prefix_ptr.dtype.element_ty).to(tl.float32) + tl.store( + prefix_ptr + row_idx * stride_prefix_m + d_offsets, + updated_prefix, + mask=d_mask, + ) + if WRITE_BLOCK: + tl.store( blocks_ptr + row_idx * stride_block_m - + source_offsets[:, None] * stride_block_r - + d_offsets[None, :] + + block_write_idx * stride_block_r + + d_offsets, + updated_prefix, + mask=d_mask, ) - block_values = tl.load( - block_ptrs, - mask=(source_mask[:, None] & ~is_prefix[:, None] & d_mask[None, :]), - other=0.0, - eviction_policy="evict_first", + # With only the prefix source, the AttnRes softmax is exactly one. + if num_blocks == 0: + mixed = updated_prefix + else: + # Reloading avoids keeping the full prefix vector live across the loop. + if HAS_DELTA: + tl.debug_barrier() + input_qk_weight = tl.load( + norm_weight_ptr + d_offsets, mask=d_mask, other=0.0 + ).to(tl.float32) * tl.load( + qk_weight_ptr + d_offsets, mask=d_mask, other=0.0 ).to(tl.float32) - values = tl.where(is_prefix[:, None], prefix[None, :], block_values) - reciprocal_std = tl.rsqrt( - tl.sum(values * values, axis=1) * (1.0 / hidden_size) + eps + max_logit = tl.full((), -float("inf"), tl.float32) + denominator = tl.zeros((), tl.float32) + mixed = tl.zeros((BLOCK_D,), tl.float32) + + num_sources = num_blocks + 1 + for source_tile in range(tl.cdiv(num_sources, BLOCK_L)): + source_offsets = source_tile * BLOCK_L + tl.arange(0, BLOCK_L) + source_mask = source_offsets < num_sources + is_prefix = source_offsets == num_blocks + block_ptrs = ( + blocks_ptr + + row_idx * stride_block_m + + source_offsets[:, None] * stride_block_r + + d_offsets[None, :] + ) + block_values = tl.load( + block_ptrs, + mask=(source_mask[:, None] & ~is_prefix[:, None] & d_mask[None, :]), + other=0.0, + eviction_policy="evict_first", + ).to(tl.float32) + values = tl.where(is_prefix[:, None], updated_prefix[None, :], block_values) + reciprocal_std = tl.rsqrt( + tl.sum(values * values, axis=1) * (1.0 / hidden_size) + eps + ) + logits = tl.sum(values * input_qk_weight[None, :], axis=1) * reciprocal_std + scores = tl.where(source_mask, logits, -float("inf")) + + new_max_logit = tl.maximum(max_logit, tl.max(scores, axis=0)) + old_scale = tl.exp(max_logit - new_max_logit) + block_scales = tl.exp(scores - new_max_logit) + denominator = denominator * old_scale + tl.sum(block_scales, axis=0) + mixed = mixed * old_scale + tl.sum(block_scales[:, None] * values, axis=0) + max_logit = new_max_logit + + mixed /= denominator + output = mixed + + if APPLY_OUTPUT_NORM: + output_reciprocal_std = tl.rsqrt( + tl.sum(tl.where(d_mask, mixed * mixed, 0.0), axis=0) * (1.0 / hidden_size) + + output_norm_eps ) - logits = tl.sum(values * input_qk_weight[None, :], axis=1) * reciprocal_std - scores = tl.where(source_mask, logits, -float("inf")) - - new_max_logit = tl.maximum(max_logit, tl.max(scores, axis=0)) - old_scale = tl.exp(max_logit - new_max_logit) - block_scales = tl.exp(scores - new_max_logit) - denominator = denominator * old_scale + tl.sum(block_scales, axis=0) - mixed = mixed * old_scale + tl.sum(block_scales[:, None] * values, axis=0) - max_logit = new_max_logit - - output = mixed / denominator + output_norm_weight = tl.load( + output_norm_weight_ptr + d_offsets, mask=d_mask, other=0.0 + ).to(tl.float32) + output = mixed * output_reciprocal_std * output_norm_weight tl.store( output_ptr + row_idx * stride_output_m + d_offsets, output, @@ -87,43 +138,53 @@ def _attn_res_kernel( def attn_res( prefix: torch.Tensor, + delta: torch.Tensor | None, blocks: torch.Tensor, norm_weight: torch.Tensor, qk_weight: torch.Tensor, + output_norm_weight: torch.Tensor | None, num_blocks: int, + block_write_idx: int, eps: float, + output_norm_eps: float, ) -> torch.Tensor: num_tokens, hidden_size = prefix.shape - assert 0 < num_blocks <= blocks.shape[1] - assert blocks.shape[0] == num_tokens - assert norm_weight.numel() == hidden_size - assert qk_weight.numel() == hidden_size assert prefix.stride(-1) == 1 + assert delta is None or delta.stride(-1) == 1 assert blocks.stride(-1) == 1 assert norm_weight.stride(-1) == 1 assert qk_weight.stride(-1) == 1 - + assert output_norm_weight is None or output_norm_weight.stride(-1) == 1 output = prefix.new_empty(prefix.shape) if num_tokens == 0: return output + # Source tiling helps decode, while one-source tiles scale better for prefill. if num_tokens >= 256 or num_blocks <= 1: block_l, num_warps = 1, 4 else: block_l, num_warps = 4, 8 _attn_res_kernel[(num_tokens,)]( prefix, + delta, blocks, norm_weight, qk_weight, + output_norm_weight, output, prefix.stride(0), + 0 if delta is None else delta.stride(0), blocks.stride(0), blocks.stride(1), output.stride(0), num_blocks, hidden_size, + block_write_idx, eps, + output_norm_eps, + HAS_DELTA=delta is not None, + WRITE_BLOCK=block_write_idx >= 0, + APPLY_OUTPUT_NORM=output_norm_weight is not None, BLOCK_L=block_l, BLOCK_D=triton.next_power_of_2(hidden_size), num_warps=num_warps,