[Kimi-K3][AMD] Fuse AttnRes state updates and norms (#50593)
Signed-off-by: Liuyinfeng01 <199041580+LiuYinfeng01@users.noreply.github.com> Co-authored-by: Liuyinfeng01 <199041580+LiuYinfeng01@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Liuyinfeng01
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
7153fd70f6
commit
7ac2ec7582
@@ -88,11 +88,15 @@ def test_amd_attn_res_matches_reference(
|
||||
|
||||
actual = attn_res(
|
||||
prefix,
|
||||
None,
|
||||
blocks,
|
||||
norm_weight,
|
||||
qk_weight,
|
||||
None,
|
||||
num_blocks,
|
||||
-1,
|
||||
eps,
|
||||
0.0,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
|
||||
@@ -100,3 +104,90 @@ def test_amd_attn_res_matches_reference(
|
||||
torch.testing.assert_close(blocks, original_blocks, atol=0, rtol=0)
|
||||
assert actual.shape == prefix.shape
|
||||
assert actual.is_contiguous()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"num_tokens",
|
||||
"num_blocks",
|
||||
"hidden_size",
|
||||
"has_delta",
|
||||
"write_block",
|
||||
"apply_output_norm",
|
||||
),
|
||||
[
|
||||
pytest.param(1, 0, 128, False, True, True, id="empty-write-norm"),
|
||||
pytest.param(7, 1, 1024, True, False, True, id="single-add-norm"),
|
||||
pytest.param(17, 5, 7168, True, True, True, id="padded-write-add"),
|
||||
pytest.param(3, 8, 7168, True, False, True, id="full-add-norm"),
|
||||
pytest.param(320, 4, 7168, True, False, False, id="prefill-add"),
|
||||
],
|
||||
)
|
||||
def test_amd_attn_res_fused_contract(
|
||||
num_tokens: int,
|
||||
num_blocks: int,
|
||||
hidden_size: int,
|
||||
has_delta: bool,
|
||||
write_block: bool,
|
||||
apply_output_norm: bool,
|
||||
) -> None:
|
||||
torch.manual_seed(42)
|
||||
eps = 1e-5
|
||||
output_eps = 2e-5
|
||||
block_capacity = 9
|
||||
prefix = _randn_with_row_padding(num_tokens, hidden_size, padding=7)
|
||||
delta = (
|
||||
_randn_with_row_padding(num_tokens, hidden_size, padding=11)
|
||||
if has_delta
|
||||
else None
|
||||
)
|
||||
blocks = _randn_with_row_padding(
|
||||
num_tokens, block_capacity, hidden_size, padding=13
|
||||
)
|
||||
norm_weight = 1 + 0.1 * torch.randn(
|
||||
hidden_size, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
qk_weight = (
|
||||
torch.randn(hidden_size, device="cuda", dtype=torch.bfloat16) / hidden_size**0.5
|
||||
)
|
||||
output_norm_weight = (
|
||||
1 + 0.1 * torch.randn(hidden_size, device="cuda", dtype=torch.bfloat16)
|
||||
if apply_output_norm
|
||||
else None
|
||||
)
|
||||
expected_prefix = prefix.clone()
|
||||
if delta is not None:
|
||||
expected_prefix = expected_prefix + delta
|
||||
values = torch.cat(
|
||||
(blocks[:, :num_blocks].clone(), expected_prefix.unsqueeze(1)), dim=1
|
||||
)
|
||||
keys = F.rms_norm(values.float(), (hidden_size,), norm_weight.float(), eps)
|
||||
probs = (keys @ qk_weight.float()).softmax(dim=-1)
|
||||
expected = torch.matmul(probs.unsqueeze(1), values.float()).squeeze(1)
|
||||
if output_norm_weight is not None:
|
||||
expected = F.rms_norm(
|
||||
expected, (hidden_size,), output_norm_weight.float(), output_eps
|
||||
)
|
||||
expected = expected.to(prefix.dtype)
|
||||
original_blocks = blocks.clone()
|
||||
block_write_idx = num_blocks if write_block else -1
|
||||
|
||||
actual = attn_res(
|
||||
prefix,
|
||||
delta,
|
||||
blocks,
|
||||
norm_weight,
|
||||
qk_weight,
|
||||
output_norm_weight,
|
||||
num_blocks,
|
||||
block_write_idx,
|
||||
eps,
|
||||
output_eps,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
|
||||
torch.testing.assert_close(prefix, expected_prefix, atol=0, rtol=0)
|
||||
if write_block:
|
||||
original_blocks[:, block_write_idx].copy_(expected_prefix)
|
||||
torch.testing.assert_close(blocks, original_blocks, atol=0, rtol=0)
|
||||
assert actual.is_contiguous()
|
||||
|
||||
@@ -141,17 +141,22 @@ def _apply_attn_res(
|
||||
proj: ReplicatedLinear,
|
||||
norm: RMSNorm,
|
||||
num_valid_blocks: int,
|
||||
*,
|
||||
delta: torch.Tensor | None = None,
|
||||
output_norm: RMSNorm | None = None,
|
||||
block_write_idx: int = -1,
|
||||
) -> torch.Tensor:
|
||||
if num_valid_blocks <= 0:
|
||||
return prefix_sum
|
||||
|
||||
return attn_res(
|
||||
prefix_sum,
|
||||
delta,
|
||||
block_residual,
|
||||
norm.weight,
|
||||
proj.weight.squeeze(0),
|
||||
None if output_norm is None else output_norm.weight,
|
||||
num_valid_blocks,
|
||||
block_write_idx,
|
||||
norm.variance_epsilon,
|
||||
0.0 if output_norm is None else output_norm.variance_epsilon,
|
||||
)
|
||||
|
||||
|
||||
@@ -610,19 +615,20 @@ class KimiDecoderLayer(nn.Module):
|
||||
self.self_attention_res_proj,
|
||||
self.self_attention_res_norm,
|
||||
self.prev_valid_blocks,
|
||||
output_norm=self.input_layernorm,
|
||||
block_write_idx=(self.block_write_idx if self.is_block_write_layer else -1),
|
||||
)
|
||||
|
||||
if self.is_block_write_layer:
|
||||
block_residual[:, self.block_write_idx, :].copy_(prefix_sum)
|
||||
prefix_sum = None
|
||||
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
hidden_states = self._run_self_attn(positions, hidden_states)
|
||||
|
||||
if prefix_sum is not None:
|
||||
prefix_sum = prefix_sum + hidden_states
|
||||
else:
|
||||
if prefix_sum is None:
|
||||
prefix_sum = hidden_states
|
||||
prefix_delta = None
|
||||
else:
|
||||
prefix_delta = hidden_states
|
||||
|
||||
mlp_valid_blocks = self.prev_valid_blocks + (
|
||||
1 if self.is_block_write_layer else 0
|
||||
@@ -633,9 +639,10 @@ class KimiDecoderLayer(nn.Module):
|
||||
self.mlp_res_proj,
|
||||
self.mlp_res_norm,
|
||||
mlp_valid_blocks,
|
||||
delta=prefix_delta,
|
||||
output_norm=self.post_attention_layernorm,
|
||||
)
|
||||
|
||||
hidden_states = self.post_attention_layernorm(hidden_states)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
prefix_sum = prefix_sum + hidden_states
|
||||
return prefix_sum, block_residual
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
# the following copyright notice:
|
||||
# Copyright (c) 2023-2026, Songlin Yang, Yu Zhang, Zhiyuan Li
|
||||
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
@@ -15,17 +16,25 @@ from vllm.triton_utils import tl, triton
|
||||
@triton.jit
|
||||
def _attn_res_kernel(
|
||||
prefix_ptr,
|
||||
delta_ptr,
|
||||
blocks_ptr,
|
||||
norm_weight_ptr,
|
||||
qk_weight_ptr,
|
||||
output_norm_weight_ptr,
|
||||
output_ptr,
|
||||
stride_prefix_m: tl.constexpr,
|
||||
stride_delta_m: tl.constexpr,
|
||||
stride_block_m: tl.constexpr,
|
||||
stride_block_r: tl.constexpr,
|
||||
stride_output_m: tl.constexpr,
|
||||
num_blocks: tl.constexpr,
|
||||
hidden_size: tl.constexpr,
|
||||
block_write_idx: tl.constexpr,
|
||||
eps: tl.constexpr,
|
||||
output_norm_eps: tl.constexpr,
|
||||
HAS_DELTA: tl.constexpr,
|
||||
WRITE_BLOCK: tl.constexpr,
|
||||
APPLY_OUTPUT_NORM: tl.constexpr,
|
||||
BLOCK_L: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
):
|
||||
@@ -33,51 +42,93 @@ def _attn_res_kernel(
|
||||
d_offsets = tl.max_contiguous(tl.arange(0, BLOCK_D), BLOCK_D)
|
||||
d_mask = d_offsets < hidden_size
|
||||
|
||||
prefix = tl.load(
|
||||
updated_prefix = tl.load(
|
||||
prefix_ptr + row_idx * stride_prefix_m + d_offsets,
|
||||
mask=d_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
input_qk_weight = tl.load(norm_weight_ptr + d_offsets, mask=d_mask, other=0.0).to(
|
||||
tl.float32
|
||||
) * tl.load(qk_weight_ptr + d_offsets, mask=d_mask, other=0.0).to(tl.float32)
|
||||
|
||||
max_logit = tl.full((), -float("inf"), tl.float32)
|
||||
denominator = tl.zeros((), tl.float32)
|
||||
mixed = tl.zeros((BLOCK_D,), tl.float32)
|
||||
num_sources = num_blocks + 1
|
||||
|
||||
for source_tile in range(tl.cdiv(num_sources, BLOCK_L)):
|
||||
source_offsets = source_tile * BLOCK_L + tl.arange(0, BLOCK_L)
|
||||
source_mask = source_offsets < num_sources
|
||||
is_prefix = source_offsets == num_blocks
|
||||
block_ptrs = (
|
||||
if HAS_DELTA:
|
||||
delta = tl.load(
|
||||
delta_ptr + row_idx * stride_delta_m + d_offsets,
|
||||
mask=d_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
updated_prefix += delta
|
||||
# Match the BF16 prefix-add result before using it as a residual source.
|
||||
updated_prefix = updated_prefix.to(prefix_ptr.dtype.element_ty).to(tl.float32)
|
||||
tl.store(
|
||||
prefix_ptr + row_idx * stride_prefix_m + d_offsets,
|
||||
updated_prefix,
|
||||
mask=d_mask,
|
||||
)
|
||||
if WRITE_BLOCK:
|
||||
tl.store(
|
||||
blocks_ptr
|
||||
+ row_idx * stride_block_m
|
||||
+ source_offsets[:, None] * stride_block_r
|
||||
+ d_offsets[None, :]
|
||||
+ block_write_idx * stride_block_r
|
||||
+ d_offsets,
|
||||
updated_prefix,
|
||||
mask=d_mask,
|
||||
)
|
||||
block_values = tl.load(
|
||||
block_ptrs,
|
||||
mask=(source_mask[:, None] & ~is_prefix[:, None] & d_mask[None, :]),
|
||||
other=0.0,
|
||||
eviction_policy="evict_first",
|
||||
# With only the prefix source, the AttnRes softmax is exactly one.
|
||||
if num_blocks == 0:
|
||||
mixed = updated_prefix
|
||||
else:
|
||||
# Reloading avoids keeping the full prefix vector live across the loop.
|
||||
if HAS_DELTA:
|
||||
tl.debug_barrier()
|
||||
input_qk_weight = tl.load(
|
||||
norm_weight_ptr + d_offsets, mask=d_mask, other=0.0
|
||||
).to(tl.float32) * tl.load(
|
||||
qk_weight_ptr + d_offsets, mask=d_mask, other=0.0
|
||||
).to(tl.float32)
|
||||
values = tl.where(is_prefix[:, None], prefix[None, :], block_values)
|
||||
reciprocal_std = tl.rsqrt(
|
||||
tl.sum(values * values, axis=1) * (1.0 / hidden_size) + eps
|
||||
max_logit = tl.full((), -float("inf"), tl.float32)
|
||||
denominator = tl.zeros((), tl.float32)
|
||||
mixed = tl.zeros((BLOCK_D,), tl.float32)
|
||||
|
||||
num_sources = num_blocks + 1
|
||||
for source_tile in range(tl.cdiv(num_sources, BLOCK_L)):
|
||||
source_offsets = source_tile * BLOCK_L + tl.arange(0, BLOCK_L)
|
||||
source_mask = source_offsets < num_sources
|
||||
is_prefix = source_offsets == num_blocks
|
||||
block_ptrs = (
|
||||
blocks_ptr
|
||||
+ row_idx * stride_block_m
|
||||
+ source_offsets[:, None] * stride_block_r
|
||||
+ d_offsets[None, :]
|
||||
)
|
||||
block_values = tl.load(
|
||||
block_ptrs,
|
||||
mask=(source_mask[:, None] & ~is_prefix[:, None] & d_mask[None, :]),
|
||||
other=0.0,
|
||||
eviction_policy="evict_first",
|
||||
).to(tl.float32)
|
||||
values = tl.where(is_prefix[:, None], updated_prefix[None, :], block_values)
|
||||
reciprocal_std = tl.rsqrt(
|
||||
tl.sum(values * values, axis=1) * (1.0 / hidden_size) + eps
|
||||
)
|
||||
logits = tl.sum(values * input_qk_weight[None, :], axis=1) * reciprocal_std
|
||||
scores = tl.where(source_mask, logits, -float("inf"))
|
||||
|
||||
new_max_logit = tl.maximum(max_logit, tl.max(scores, axis=0))
|
||||
old_scale = tl.exp(max_logit - new_max_logit)
|
||||
block_scales = tl.exp(scores - new_max_logit)
|
||||
denominator = denominator * old_scale + tl.sum(block_scales, axis=0)
|
||||
mixed = mixed * old_scale + tl.sum(block_scales[:, None] * values, axis=0)
|
||||
max_logit = new_max_logit
|
||||
|
||||
mixed /= denominator
|
||||
output = mixed
|
||||
|
||||
if APPLY_OUTPUT_NORM:
|
||||
output_reciprocal_std = tl.rsqrt(
|
||||
tl.sum(tl.where(d_mask, mixed * mixed, 0.0), axis=0) * (1.0 / hidden_size)
|
||||
+ output_norm_eps
|
||||
)
|
||||
logits = tl.sum(values * input_qk_weight[None, :], axis=1) * reciprocal_std
|
||||
scores = tl.where(source_mask, logits, -float("inf"))
|
||||
|
||||
new_max_logit = tl.maximum(max_logit, tl.max(scores, axis=0))
|
||||
old_scale = tl.exp(max_logit - new_max_logit)
|
||||
block_scales = tl.exp(scores - new_max_logit)
|
||||
denominator = denominator * old_scale + tl.sum(block_scales, axis=0)
|
||||
mixed = mixed * old_scale + tl.sum(block_scales[:, None] * values, axis=0)
|
||||
max_logit = new_max_logit
|
||||
|
||||
output = mixed / denominator
|
||||
output_norm_weight = tl.load(
|
||||
output_norm_weight_ptr + d_offsets, mask=d_mask, other=0.0
|
||||
).to(tl.float32)
|
||||
output = mixed * output_reciprocal_std * output_norm_weight
|
||||
tl.store(
|
||||
output_ptr + row_idx * stride_output_m + d_offsets,
|
||||
output,
|
||||
@@ -87,43 +138,53 @@ def _attn_res_kernel(
|
||||
|
||||
def attn_res(
|
||||
prefix: torch.Tensor,
|
||||
delta: torch.Tensor | None,
|
||||
blocks: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
qk_weight: torch.Tensor,
|
||||
output_norm_weight: torch.Tensor | None,
|
||||
num_blocks: int,
|
||||
block_write_idx: int,
|
||||
eps: float,
|
||||
output_norm_eps: float,
|
||||
) -> torch.Tensor:
|
||||
num_tokens, hidden_size = prefix.shape
|
||||
assert 0 < num_blocks <= blocks.shape[1]
|
||||
assert blocks.shape[0] == num_tokens
|
||||
assert norm_weight.numel() == hidden_size
|
||||
assert qk_weight.numel() == hidden_size
|
||||
assert prefix.stride(-1) == 1
|
||||
assert delta is None or delta.stride(-1) == 1
|
||||
assert blocks.stride(-1) == 1
|
||||
assert norm_weight.stride(-1) == 1
|
||||
assert qk_weight.stride(-1) == 1
|
||||
|
||||
assert output_norm_weight is None or output_norm_weight.stride(-1) == 1
|
||||
output = prefix.new_empty(prefix.shape)
|
||||
if num_tokens == 0:
|
||||
return output
|
||||
|
||||
# Source tiling helps decode, while one-source tiles scale better for prefill.
|
||||
if num_tokens >= 256 or num_blocks <= 1:
|
||||
block_l, num_warps = 1, 4
|
||||
else:
|
||||
block_l, num_warps = 4, 8
|
||||
_attn_res_kernel[(num_tokens,)](
|
||||
prefix,
|
||||
delta,
|
||||
blocks,
|
||||
norm_weight,
|
||||
qk_weight,
|
||||
output_norm_weight,
|
||||
output,
|
||||
prefix.stride(0),
|
||||
0 if delta is None else delta.stride(0),
|
||||
blocks.stride(0),
|
||||
blocks.stride(1),
|
||||
output.stride(0),
|
||||
num_blocks,
|
||||
hidden_size,
|
||||
block_write_idx,
|
||||
eps,
|
||||
output_norm_eps,
|
||||
HAS_DELTA=delta is not None,
|
||||
WRITE_BLOCK=block_write_idx >= 0,
|
||||
APPLY_OUTPUT_NORM=output_norm_weight is not None,
|
||||
BLOCK_L=block_l,
|
||||
BLOCK_D=triton.next_power_of_2(hidden_size),
|
||||
num_warps=num_warps,
|
||||
|
||||
Reference in New Issue
Block a user