[K3 Perf] Optimize k3 mamba metadata preparation, 6.6~7.6x kernel performance improvement (#52388)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -271,6 +271,24 @@ class MambaHybridModelState(DefaultModelState):
|
||||
)
|
||||
num_decode_draft_tokens_cpu = torch.from_numpy(num_decode_draft_tokens_np)
|
||||
|
||||
if self._align_mode:
|
||||
mamba_group_ids, _ = self._get_mamba_group_info(kv_cache_config)
|
||||
aligned_index_builders = []
|
||||
for group_idx, group_id in enumerate(mamba_group_ids):
|
||||
for group in attn_groups[group_id]:
|
||||
builder = group.get_metadata_builder(0)
|
||||
if hasattr(builder, "mamba_aligned_state_indices"):
|
||||
aligned_index_builders.append((group_idx, builder))
|
||||
if aligned_index_builders:
|
||||
ctx = self._ensure_align_ctx(
|
||||
kv_cache_config, mamba_group_ids, block_tables
|
||||
)
|
||||
all_group_indices = ctx.compute_aligned_state_indices(
|
||||
input_batch.seq_lens, num_reqs
|
||||
)
|
||||
for group_idx, builder in aligned_index_builders:
|
||||
builder.mamba_aligned_state_indices = all_group_indices[group_idx]
|
||||
|
||||
mamba_attn_metadata = MambaHybridAttnMetadata(
|
||||
is_prefilling=is_prefilling,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
|
||||
Reference in New Issue
Block a user