[2/N][Core] support partial prefix cache hit for hybrid model (#46384)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Jiangyun Zhu
2026-07-12 05:37:51 +00:00
committed by GitHub
co-authored by Yifan Qiao
parent 8e981630c9
commit 481e481be7
21 changed files with 1653 additions and 177 deletions
+7
View File
@@ -230,6 +230,7 @@ from .utils import (
KVBlockZeroer,
add_kv_sharing_layers_to_kv_cache_groups,
bind_kv_cache,
copy_kv_cache_blocks_inplace,
prepare_kernel_block_sizes,
sanity_check_mm_encoder_outputs,
)
@@ -1188,6 +1189,12 @@ class GPUModelRunner(
# stale NaN/data from corrupting attention or SSM computation.
if scheduler_output.new_block_ids_to_zero:
self._zero_block_ids(scheduler_output.new_block_ids_to_zero)
if scheduler_output.kv_cache_block_copies:
copy_kv_cache_blocks_inplace(
self.kv_caches,
self.kv_cache_config.num_blocks,
scheduler_output.kv_cache_block_copies,
)
# Free the cached encoder outputs.
for mm_hash in scheduler_output.free_encoder_mm_hashes: