[Kimi K3 Perf] Avoid KDA mixed-batch gather/scatter, 5.2%~7.7% E2E Throughput Improvement (#56159)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-09-10 17:57:01 +08:00
committed by GitHub
parent c9355e25e8
commit 86aca66191
6 changed files with 66 additions and 5 deletions
+7
View File
@@ -330,6 +330,7 @@ def test_chunk_kda_fused_gate_cumsum_matches_unfused(
cu_seqlens=cu_seqlens_t,
use_qk_l2norm_in_kernel=True,
)
output = torch.empty_like(v)
new_o, new_ht = chunk_kda_with_fused_gate(
q=q.clone(),
k=k.clone(),
@@ -343,8 +344,10 @@ def test_chunk_kda_fused_gate_cumsum_matches_unfused(
output_final_state=True,
cu_seqlens=cu_seqlens_t,
use_qk_l2norm_in_kernel=True,
out=output,
)
assert new_o.data_ptr() == output.data_ptr()
assert_close("o", old_o, new_o, 1e-3, err_atol=1e-3)
assert_close("ht", old_ht, new_ht, 1e-3, err_atol=1e-3)
@@ -442,6 +445,7 @@ def test_packed_kda_decode_correctness(
use_qk_l2norm_in_kernel=True,
)
packed_state = state
packed_output = torch.empty_like(dense_out)
packed_out, _ = PACKED_DECODE_IMPLS[impl](
mixed_qkv=mixed_qkv,
raw_g=raw_g,
@@ -451,8 +455,11 @@ def test_packed_kda_decode_correctness(
lower_bound=lower_bound,
initial_state=packed_state,
state_indices=state_indices,
**({"out": packed_output} if impl == "nvidia" else {}),
)
if impl == "nvidia":
assert packed_out.data_ptr() == packed_output.data_ptr()
assert_close("o", dense_out, packed_out, 1e-3, err_atol=1e-3)
assert_close("ht", dense_state, packed_state, 1e-3, err_atol=1e-3)
@@ -512,6 +512,8 @@ def test_mixed_regular_and_spec_decode_uses_packed_decode_metadata():
assert actual.non_spec_query_start_loc is None
torch.testing.assert_close(actual.non_spec_token_indx, torch.tensor([0, 1]))
torch.testing.assert_close(actual.spec_token_indx, torch.tensor([2, 3, 4]))
assert actual.non_spec_token_start == 0
assert actual.spec_token_start == 2
torch.testing.assert_close(
actual.spec_query_start_loc,
torch.tensor([0, 3], dtype=torch.int32),
@@ -539,6 +541,8 @@ def test_mixed_regular_and_spec_decode_excludes_request_padding():
assert actual.non_spec_state_indices_tensor.shape == (1,)
torch.testing.assert_close(actual.non_spec_token_indx, torch.tensor([0]))
torch.testing.assert_close(actual.spec_token_indx, torch.tensor([1, 2, 3]))
assert actual.non_spec_token_start == 0
assert actual.spec_token_start == 1
@pytest.mark.parametrize("mamba_cache_mode", ["none", "align"])