[Perf][Kimi-K3] Cut MLA decode concat/cache epilogue latency (#54896)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
c7e6e36fa9
commit
9509fc8ae6
@@ -285,3 +285,26 @@ def test_decode_epilogue_preserves_nope_path() -> None:
|
||||
torch.testing.assert_close(
|
||||
_cache_rows(cache, slots), torch.cat((kv_c, k_pe), dim=-1)
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
@pytest.mark.parametrize("num_tokens", [64, 65])
|
||||
def test_decode_epilogue_row_split_boundary(num_tokens: int) -> None:
|
||||
"""Both sides of the M-based warps-per-row dispatch produce the same rows."""
|
||||
torch.manual_seed(3)
|
||||
num_blocks = (num_tokens + _BLOCK_SIZE - 1) // _BLOCK_SIZE
|
||||
slots = torch.randperm(num_blocks * _BLOCK_SIZE, device="cuda")[:num_tokens].long()
|
||||
ql_nope = _randn(num_tokens, _NUM_HEADS, 512)
|
||||
q_pe = _randn(num_tokens, _NUM_HEADS, 64)
|
||||
kv_c = _randn(num_tokens, 512)
|
||||
k_pe = _randn(num_tokens, 64)
|
||||
cache = torch.zeros(num_blocks, _BLOCK_SIZE, 576, device="cuda", dtype=_DTYPE)
|
||||
|
||||
q_actual = fused_mla_decode_q_concat_kv_cache_insert(
|
||||
ql_nope, q_pe, kv_c, k_pe, cache, slots
|
||||
)
|
||||
|
||||
torch.testing.assert_close(q_actual, torch.cat((ql_nope, q_pe), dim=-1))
|
||||
torch.testing.assert_close(
|
||||
_cache_rows(cache, slots), torch.cat((kv_c, k_pe), dim=-1)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user