[Kimi-K3][AMD] Fuse AttnRes state updates and norms (#50593)

Signed-off-by: Liuyinfeng01 <199041580+LiuYinfeng01@users.noreply.github.com>
Co-authored-by: Liuyinfeng01 <199041580+LiuYinfeng01@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
yinfengLiu
2026-08-04 12:32:13 -04:00
committed by GitHub
co-authored by Liuyinfeng01 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 7153fd70f6
commit 7ac2ec7582
3 changed files with 209 additions and 50 deletions
+91
View File
@@ -88,11 +88,15 @@ def test_amd_attn_res_matches_reference(
actual = attn_res(
prefix,
None,
blocks,
norm_weight,
qk_weight,
None,
num_blocks,
-1,
eps,
0.0,
)
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
@@ -100,3 +104,90 @@ def test_amd_attn_res_matches_reference(
torch.testing.assert_close(blocks, original_blocks, atol=0, rtol=0)
assert actual.shape == prefix.shape
assert actual.is_contiguous()
@pytest.mark.parametrize(
(
"num_tokens",
"num_blocks",
"hidden_size",
"has_delta",
"write_block",
"apply_output_norm",
),
[
pytest.param(1, 0, 128, False, True, True, id="empty-write-norm"),
pytest.param(7, 1, 1024, True, False, True, id="single-add-norm"),
pytest.param(17, 5, 7168, True, True, True, id="padded-write-add"),
pytest.param(3, 8, 7168, True, False, True, id="full-add-norm"),
pytest.param(320, 4, 7168, True, False, False, id="prefill-add"),
],
)
def test_amd_attn_res_fused_contract(
num_tokens: int,
num_blocks: int,
hidden_size: int,
has_delta: bool,
write_block: bool,
apply_output_norm: bool,
) -> None:
torch.manual_seed(42)
eps = 1e-5
output_eps = 2e-5
block_capacity = 9
prefix = _randn_with_row_padding(num_tokens, hidden_size, padding=7)
delta = (
_randn_with_row_padding(num_tokens, hidden_size, padding=11)
if has_delta
else None
)
blocks = _randn_with_row_padding(
num_tokens, block_capacity, hidden_size, padding=13
)
norm_weight = 1 + 0.1 * torch.randn(
hidden_size, device="cuda", dtype=torch.bfloat16
)
qk_weight = (
torch.randn(hidden_size, device="cuda", dtype=torch.bfloat16) / hidden_size**0.5
)
output_norm_weight = (
1 + 0.1 * torch.randn(hidden_size, device="cuda", dtype=torch.bfloat16)
if apply_output_norm
else None
)
expected_prefix = prefix.clone()
if delta is not None:
expected_prefix = expected_prefix + delta
values = torch.cat(
(blocks[:, :num_blocks].clone(), expected_prefix.unsqueeze(1)), dim=1
)
keys = F.rms_norm(values.float(), (hidden_size,), norm_weight.float(), eps)
probs = (keys @ qk_weight.float()).softmax(dim=-1)
expected = torch.matmul(probs.unsqueeze(1), values.float()).squeeze(1)
if output_norm_weight is not None:
expected = F.rms_norm(
expected, (hidden_size,), output_norm_weight.float(), output_eps
)
expected = expected.to(prefix.dtype)
original_blocks = blocks.clone()
block_write_idx = num_blocks if write_block else -1
actual = attn_res(
prefix,
delta,
blocks,
norm_weight,
qk_weight,
output_norm_weight,
num_blocks,
block_write_idx,
eps,
output_eps,
)
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
torch.testing.assert_close(prefix, expected_prefix, atol=0, rtol=0)
if write_block:
original_blocks[:, block_write_idx].copy_(expected_prefix)
torch.testing.assert_close(blocks, original_blocks, atol=0, rtol=0)
assert actual.is_contiguous()
+16 -9
View File
@@ -141,17 +141,22 @@ def _apply_attn_res(
proj: ReplicatedLinear,
norm: RMSNorm,
num_valid_blocks: int,
*,
delta: torch.Tensor | None = None,
output_norm: RMSNorm | None = None,
block_write_idx: int = -1,
) -> torch.Tensor:
if num_valid_blocks <= 0:
return prefix_sum
return attn_res(
prefix_sum,
delta,
block_residual,
norm.weight,
proj.weight.squeeze(0),
None if output_norm is None else output_norm.weight,
num_valid_blocks,
block_write_idx,
norm.variance_epsilon,
0.0 if output_norm is None else output_norm.variance_epsilon,
)
@@ -610,19 +615,20 @@ class KimiDecoderLayer(nn.Module):
self.self_attention_res_proj,
self.self_attention_res_norm,
self.prev_valid_blocks,
output_norm=self.input_layernorm,
block_write_idx=(self.block_write_idx if self.is_block_write_layer else -1),
)
if self.is_block_write_layer:
block_residual[:, self.block_write_idx, :].copy_(prefix_sum)
prefix_sum = None
hidden_states = self.input_layernorm(hidden_states)
hidden_states = self._run_self_attn(positions, hidden_states)
if prefix_sum is not None:
prefix_sum = prefix_sum + hidden_states
else:
if prefix_sum is None:
prefix_sum = hidden_states
prefix_delta = None
else:
prefix_delta = hidden_states
mlp_valid_blocks = self.prev_valid_blocks + (
1 if self.is_block_write_layer else 0
@@ -633,9 +639,10 @@ class KimiDecoderLayer(nn.Module):
self.mlp_res_proj,
self.mlp_res_norm,
mlp_valid_blocks,
delta=prefix_delta,
output_norm=self.post_attention_layernorm,
)
hidden_states = self.post_attention_layernorm(hidden_states)
hidden_states = self.mlp(hidden_states)
prefix_sum = prefix_sum + hidden_states
return prefix_sum, block_residual
+102 -41
View File
@@ -7,6 +7,7 @@
# the following copyright notice:
# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
import torch
from vllm.triton_utils import tl, triton
@@ -15,17 +16,25 @@ from vllm.triton_utils import tl, triton
@triton.jit
def _attn_res_kernel(
prefix_ptr,
delta_ptr,
blocks_ptr,
norm_weight_ptr,
qk_weight_ptr,
output_norm_weight_ptr,
output_ptr,
stride_prefix_m: tl.constexpr,
stride_delta_m: tl.constexpr,
stride_block_m: tl.constexpr,
stride_block_r: tl.constexpr,
stride_output_m: tl.constexpr,
num_blocks: tl.constexpr,
hidden_size: tl.constexpr,
block_write_idx: tl.constexpr,
eps: tl.constexpr,
output_norm_eps: tl.constexpr,
HAS_DELTA: tl.constexpr,
WRITE_BLOCK: tl.constexpr,
APPLY_OUTPUT_NORM: tl.constexpr,
BLOCK_L: tl.constexpr,
BLOCK_D: tl.constexpr,
):
@@ -33,51 +42,93 @@ def _attn_res_kernel(
d_offsets = tl.max_contiguous(tl.arange(0, BLOCK_D), BLOCK_D)
d_mask = d_offsets < hidden_size
prefix = tl.load(
updated_prefix = tl.load(
prefix_ptr + row_idx * stride_prefix_m + d_offsets,
mask=d_mask,
other=0.0,
).to(tl.float32)
input_qk_weight = tl.load(norm_weight_ptr + d_offsets, mask=d_mask, other=0.0).to(
tl.float32
) * tl.load(qk_weight_ptr + d_offsets, mask=d_mask, other=0.0).to(tl.float32)
max_logit = tl.full((), -float("inf"), tl.float32)
denominator = tl.zeros((), tl.float32)
mixed = tl.zeros((BLOCK_D,), tl.float32)
num_sources = num_blocks + 1
for source_tile in range(tl.cdiv(num_sources, BLOCK_L)):
source_offsets = source_tile * BLOCK_L + tl.arange(0, BLOCK_L)
source_mask = source_offsets < num_sources
is_prefix = source_offsets == num_blocks
block_ptrs = (
if HAS_DELTA:
delta = tl.load(
delta_ptr + row_idx * stride_delta_m + d_offsets,
mask=d_mask,
other=0.0,
).to(tl.float32)
updated_prefix += delta
# Match the BF16 prefix-add result before using it as a residual source.
updated_prefix = updated_prefix.to(prefix_ptr.dtype.element_ty).to(tl.float32)
tl.store(
prefix_ptr + row_idx * stride_prefix_m + d_offsets,
updated_prefix,
mask=d_mask,
)
if WRITE_BLOCK:
tl.store(
blocks_ptr
+ row_idx * stride_block_m
+ source_offsets[:, None] * stride_block_r
+ d_offsets[None, :]
+ block_write_idx * stride_block_r
+ d_offsets,
updated_prefix,
mask=d_mask,
)
block_values = tl.load(
block_ptrs,
mask=(source_mask[:, None] & ~is_prefix[:, None] & d_mask[None, :]),
other=0.0,
eviction_policy="evict_first",
# With only the prefix source, the AttnRes softmax is exactly one.
if num_blocks == 0:
mixed = updated_prefix
else:
# Reloading avoids keeping the full prefix vector live across the loop.
if HAS_DELTA:
tl.debug_barrier()
input_qk_weight = tl.load(
norm_weight_ptr + d_offsets, mask=d_mask, other=0.0
).to(tl.float32) * tl.load(
qk_weight_ptr + d_offsets, mask=d_mask, other=0.0
).to(tl.float32)
values = tl.where(is_prefix[:, None], prefix[None, :], block_values)
reciprocal_std = tl.rsqrt(
tl.sum(values * values, axis=1) * (1.0 / hidden_size) + eps
max_logit = tl.full((), -float("inf"), tl.float32)
denominator = tl.zeros((), tl.float32)
mixed = tl.zeros((BLOCK_D,), tl.float32)
num_sources = num_blocks + 1
for source_tile in range(tl.cdiv(num_sources, BLOCK_L)):
source_offsets = source_tile * BLOCK_L + tl.arange(0, BLOCK_L)
source_mask = source_offsets < num_sources
is_prefix = source_offsets == num_blocks
block_ptrs = (
blocks_ptr
+ row_idx * stride_block_m
+ source_offsets[:, None] * stride_block_r
+ d_offsets[None, :]
)
block_values = tl.load(
block_ptrs,
mask=(source_mask[:, None] & ~is_prefix[:, None] & d_mask[None, :]),
other=0.0,
eviction_policy="evict_first",
).to(tl.float32)
values = tl.where(is_prefix[:, None], updated_prefix[None, :], block_values)
reciprocal_std = tl.rsqrt(
tl.sum(values * values, axis=1) * (1.0 / hidden_size) + eps
)
logits = tl.sum(values * input_qk_weight[None, :], axis=1) * reciprocal_std
scores = tl.where(source_mask, logits, -float("inf"))
new_max_logit = tl.maximum(max_logit, tl.max(scores, axis=0))
old_scale = tl.exp(max_logit - new_max_logit)
block_scales = tl.exp(scores - new_max_logit)
denominator = denominator * old_scale + tl.sum(block_scales, axis=0)
mixed = mixed * old_scale + tl.sum(block_scales[:, None] * values, axis=0)
max_logit = new_max_logit
mixed /= denominator
output = mixed
if APPLY_OUTPUT_NORM:
output_reciprocal_std = tl.rsqrt(
tl.sum(tl.where(d_mask, mixed * mixed, 0.0), axis=0) * (1.0 / hidden_size)
+ output_norm_eps
)
logits = tl.sum(values * input_qk_weight[None, :], axis=1) * reciprocal_std
scores = tl.where(source_mask, logits, -float("inf"))
new_max_logit = tl.maximum(max_logit, tl.max(scores, axis=0))
old_scale = tl.exp(max_logit - new_max_logit)
block_scales = tl.exp(scores - new_max_logit)
denominator = denominator * old_scale + tl.sum(block_scales, axis=0)
mixed = mixed * old_scale + tl.sum(block_scales[:, None] * values, axis=0)
max_logit = new_max_logit
output = mixed / denominator
output_norm_weight = tl.load(
output_norm_weight_ptr + d_offsets, mask=d_mask, other=0.0
).to(tl.float32)
output = mixed * output_reciprocal_std * output_norm_weight
tl.store(
output_ptr + row_idx * stride_output_m + d_offsets,
output,
@@ -87,43 +138,53 @@ def _attn_res_kernel(
def attn_res(
prefix: torch.Tensor,
delta: torch.Tensor | None,
blocks: torch.Tensor,
norm_weight: torch.Tensor,
qk_weight: torch.Tensor,
output_norm_weight: torch.Tensor | None,
num_blocks: int,
block_write_idx: int,
eps: float,
output_norm_eps: float,
) -> torch.Tensor:
num_tokens, hidden_size = prefix.shape
assert 0 < num_blocks <= blocks.shape[1]
assert blocks.shape[0] == num_tokens
assert norm_weight.numel() == hidden_size
assert qk_weight.numel() == hidden_size
assert prefix.stride(-1) == 1
assert delta is None or delta.stride(-1) == 1
assert blocks.stride(-1) == 1
assert norm_weight.stride(-1) == 1
assert qk_weight.stride(-1) == 1
assert output_norm_weight is None or output_norm_weight.stride(-1) == 1
output = prefix.new_empty(prefix.shape)
if num_tokens == 0:
return output
# Source tiling helps decode, while one-source tiles scale better for prefill.
if num_tokens >= 256 or num_blocks <= 1:
block_l, num_warps = 1, 4
else:
block_l, num_warps = 4, 8
_attn_res_kernel[(num_tokens,)](
prefix,
delta,
blocks,
norm_weight,
qk_weight,
output_norm_weight,
output,
prefix.stride(0),
0 if delta is None else delta.stride(0),
blocks.stride(0),
blocks.stride(1),
output.stride(0),
num_blocks,
hidden_size,
block_write_idx,
eps,
output_norm_eps,
HAS_DELTA=delta is not None,
WRITE_BLOCK=block_write_idx >= 0,
APPLY_OUTPUT_NORM=output_norm_weight is not None,
BLOCK_L=block_l,
BLOCK_D=triton.next_power_of_2(hidden_size),
num_warps=num_warps,