[Kimi K3 Perf] Avoid KDA mixed-batch gather/scatter, 5.2%~7.7% E2E Throughput Improvement (#56159)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -330,6 +330,7 @@ def test_chunk_kda_fused_gate_cumsum_matches_unfused(
|
||||
cu_seqlens=cu_seqlens_t,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
output = torch.empty_like(v)
|
||||
new_o, new_ht = chunk_kda_with_fused_gate(
|
||||
q=q.clone(),
|
||||
k=k.clone(),
|
||||
@@ -343,8 +344,10 @@ def test_chunk_kda_fused_gate_cumsum_matches_unfused(
|
||||
output_final_state=True,
|
||||
cu_seqlens=cu_seqlens_t,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
out=output,
|
||||
)
|
||||
|
||||
assert new_o.data_ptr() == output.data_ptr()
|
||||
assert_close("o", old_o, new_o, 1e-3, err_atol=1e-3)
|
||||
assert_close("ht", old_ht, new_ht, 1e-3, err_atol=1e-3)
|
||||
|
||||
@@ -442,6 +445,7 @@ def test_packed_kda_decode_correctness(
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
packed_state = state
|
||||
packed_output = torch.empty_like(dense_out)
|
||||
packed_out, _ = PACKED_DECODE_IMPLS[impl](
|
||||
mixed_qkv=mixed_qkv,
|
||||
raw_g=raw_g,
|
||||
@@ -451,8 +455,11 @@ def test_packed_kda_decode_correctness(
|
||||
lower_bound=lower_bound,
|
||||
initial_state=packed_state,
|
||||
state_indices=state_indices,
|
||||
**({"out": packed_output} if impl == "nvidia" else {}),
|
||||
)
|
||||
|
||||
if impl == "nvidia":
|
||||
assert packed_out.data_ptr() == packed_output.data_ptr()
|
||||
assert_close("o", dense_out, packed_out, 1e-3, err_atol=1e-3)
|
||||
assert_close("ht", dense_state, packed_state, 1e-3, err_atol=1e-3)
|
||||
|
||||
|
||||
@@ -512,6 +512,8 @@ def test_mixed_regular_and_spec_decode_uses_packed_decode_metadata():
|
||||
assert actual.non_spec_query_start_loc is None
|
||||
torch.testing.assert_close(actual.non_spec_token_indx, torch.tensor([0, 1]))
|
||||
torch.testing.assert_close(actual.spec_token_indx, torch.tensor([2, 3, 4]))
|
||||
assert actual.non_spec_token_start == 0
|
||||
assert actual.spec_token_start == 2
|
||||
torch.testing.assert_close(
|
||||
actual.spec_query_start_loc,
|
||||
torch.tensor([0, 3], dtype=torch.int32),
|
||||
@@ -539,6 +541,8 @@ def test_mixed_regular_and_spec_decode_excludes_request_padding():
|
||||
assert actual.non_spec_state_indices_tensor.shape == (1,)
|
||||
torch.testing.assert_close(actual.non_spec_token_indx, torch.tensor([0]))
|
||||
torch.testing.assert_close(actual.spec_token_indx, torch.tensor([1, 2, 3]))
|
||||
assert actual.non_spec_token_start == 0
|
||||
assert actual.spec_token_start == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mamba_cache_mode", ["none", "align"])
|
||||
|
||||
Reference in New Issue
Block a user