[K3 Perf] Optimize k3 mamba metadata preparation, 6.6~7.6x kernel performance improvement (#52388)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-08-25 11:00:28 -04:00
committed by GitHub
parent d5cadcee86
commit 41729fc53b
4 changed files with 187 additions and 6 deletions
+37
View File
@@ -330,6 +330,13 @@ def test_recoverssm_spec_uses_one_state_slot_and_current_window(
use_recoverssm=True,
)
assert isinstance(builder, KimiK3KDAMetadataBuilder)
if mamba_cache_mode == "align":
builder.mamba_aligned_state_indices = mamba_get_block_table_tensor(
common_attn_metadata.block_table_tensor,
common_attn_metadata.seq_lens,
builder.kv_cache_spec,
mamba_cache_mode,
)
context = builder.recoverssm_context
assert context is not None
actual = builder.build(
@@ -506,6 +513,36 @@ def test_kimi_k3_kda_backend_uses_private_metadata_builder():
assert issubclass(KimiK3KDAMetadataBuilder, GDNAttentionMetadataBuilder)
def test_kimi_k3_metadata_uses_precomputed_aligned_state_indices():
batch = BatchSpec(seq_lens=[40, 30], query_lens=[1, 1])
common_attn_metadata = create_common_attn_metadata(
batch, BLOCK_SIZE, DEVICE
).replace(is_prefilling=torch.tensor([False, False]))
builder = _make_builder(
KimiK3KDAMetadataBuilder,
num_speculative_tokens=2,
full_cuda_graph=False,
mamba_cache_mode="align",
)
assert isinstance(builder, KimiK3KDAMetadataBuilder)
precomputed_indices = torch.tensor(
[
[101, 102, 103],
[201, 202, 203],
[301, 302, 303],
],
dtype=torch.int32,
)
builder.mamba_aligned_state_indices = precomputed_indices
metadata = builder.build(0, common_attn_metadata)
torch.testing.assert_close(
metadata.non_spec_state_indices_tensor,
precomputed_indices[: batch.batch_size, 0],
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_stage_spec_decode_metadata_matches_pytorch():
device = torch.device("cuda")