[K3 Perf] Optimize k3 mamba metadata preparation, 6.6~7.6x kernel performance improvement (#52388)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -330,6 +330,13 @@ def test_recoverssm_spec_uses_one_state_slot_and_current_window(
|
||||
use_recoverssm=True,
|
||||
)
|
||||
assert isinstance(builder, KimiK3KDAMetadataBuilder)
|
||||
if mamba_cache_mode == "align":
|
||||
builder.mamba_aligned_state_indices = mamba_get_block_table_tensor(
|
||||
common_attn_metadata.block_table_tensor,
|
||||
common_attn_metadata.seq_lens,
|
||||
builder.kv_cache_spec,
|
||||
mamba_cache_mode,
|
||||
)
|
||||
context = builder.recoverssm_context
|
||||
assert context is not None
|
||||
actual = builder.build(
|
||||
@@ -506,6 +513,36 @@ def test_kimi_k3_kda_backend_uses_private_metadata_builder():
|
||||
assert issubclass(KimiK3KDAMetadataBuilder, GDNAttentionMetadataBuilder)
|
||||
|
||||
|
||||
def test_kimi_k3_metadata_uses_precomputed_aligned_state_indices():
|
||||
batch = BatchSpec(seq_lens=[40, 30], query_lens=[1, 1])
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch, BLOCK_SIZE, DEVICE
|
||||
).replace(is_prefilling=torch.tensor([False, False]))
|
||||
builder = _make_builder(
|
||||
KimiK3KDAMetadataBuilder,
|
||||
num_speculative_tokens=2,
|
||||
full_cuda_graph=False,
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
assert isinstance(builder, KimiK3KDAMetadataBuilder)
|
||||
precomputed_indices = torch.tensor(
|
||||
[
|
||||
[101, 102, 103],
|
||||
[201, 202, 203],
|
||||
[301, 302, 303],
|
||||
],
|
||||
dtype=torch.int32,
|
||||
)
|
||||
builder.mamba_aligned_state_indices = precomputed_indices
|
||||
|
||||
metadata = builder.build(0, common_attn_metadata)
|
||||
|
||||
torch.testing.assert_close(
|
||||
metadata.non_spec_state_indices_tensor,
|
||||
precomputed_indices[: batch.batch_size, 0],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_stage_spec_decode_metadata_matches_pytorch():
|
||||
device = torch.device("cuda")
|
||||
|
||||
Reference in New Issue
Block a user