[K3 Perf] Optimize k3 mamba metadata preparation, 6.6~7.6x kernel performance improvement (#52388)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -306,6 +306,8 @@ class KimiK3KDAMetadata(GDNAttentionMetadata, RecoverSSMMetadata):
|
||||
|
||||
|
||||
class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
|
||||
mamba_aligned_state_indices: torch.Tensor | None = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_spec: MambaSpec,
|
||||
@@ -367,12 +369,22 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
|
||||
# offsets = torch.arange(1 + num_speculative_blocks, dtype=torch.int32)
|
||||
# indices = (start[:, None] + offsets).to(torch.int64)
|
||||
# block_table_tensor = torch.gather(block_table, 1, indices)
|
||||
block_table_tensor = _mamba_get_block_table_tensor(
|
||||
m.block_table_tensor,
|
||||
m.seq_lens,
|
||||
self.kv_cache_spec,
|
||||
self.vllm_config.cache_config.mamba_cache_mode,
|
||||
)
|
||||
if self.vllm_config.cache_config.mamba_cache_mode == "align":
|
||||
if self.mamba_aligned_state_indices is not None:
|
||||
block_table_tensor = self.mamba_aligned_state_indices[: m.num_reqs]
|
||||
else:
|
||||
assert not self.vllm_config.use_v2_model_runner, (
|
||||
"Aligned Mamba state indices must be precomputed"
|
||||
)
|
||||
# TODO: remove this MRV1 fallback once MRV2 is the default runner.
|
||||
block_table_tensor = _mamba_get_block_table_tensor(
|
||||
m.block_table_tensor,
|
||||
m.seq_lens,
|
||||
self.kv_cache_spec,
|
||||
self.vllm_config.cache_config.mamba_cache_mode,
|
||||
)
|
||||
else:
|
||||
block_table_tensor = m.block_table_tensor
|
||||
|
||||
if not self.use_spec_decode or num_decode_draft_tokens_cpu is None:
|
||||
spec_sequence_masks_cpu = None
|
||||
|
||||
Reference in New Issue
Block a user