[Perf][Kimi-K3] Cut MLA decode concat/cache epilogue latency (#54896)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Yongye Zhu
2026-09-03 15:39:58 -07:00
committed by GitHub
co-authored by Claude Fable 5
parent c7e6e36fa9
commit 9509fc8ae6
2 changed files with 121 additions and 36 deletions
@@ -285,3 +285,26 @@ def test_decode_epilogue_preserves_nope_path() -> None:
torch.testing.assert_close(
_cache_rows(cache, slots), torch.cat((kv_c, k_pe), dim=-1)
)
@torch.inference_mode()
@pytest.mark.parametrize("num_tokens", [64, 65])
def test_decode_epilogue_row_split_boundary(num_tokens: int) -> None:
"""Both sides of the M-based warps-per-row dispatch produce the same rows."""
torch.manual_seed(3)
num_blocks = (num_tokens + _BLOCK_SIZE - 1) // _BLOCK_SIZE
slots = torch.randperm(num_blocks * _BLOCK_SIZE, device="cuda")[:num_tokens].long()
ql_nope = _randn(num_tokens, _NUM_HEADS, 512)
q_pe = _randn(num_tokens, _NUM_HEADS, 64)
kv_c = _randn(num_tokens, 512)
k_pe = _randn(num_tokens, 64)
cache = torch.zeros(num_blocks, _BLOCK_SIZE, 576, device="cuda", dtype=_DTYPE)
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
ql_nope, q_pe, kv_c, k_pe, cache, slots
)
torch.testing.assert_close(q_actual, torch.cat((ql_nope, q_pe), dim=-1))
torch.testing.assert_close(
_cache_rows(cache, slots), torch.cat((kv_c, k_pe), dim=-1)
)