[Bugfix] Index Mamba align state copies by request slot in MRV2 (#60210)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Karl0007 <44745209+Karl0007@users.noreply.github.com> Co-authored-by: Yongqin Wang <yongqinwang@roblox.com>
This commit is contained in:
co-authored by
Karl0007
Yongqin Wang
parent
956960f85f
commit
8a26869bca
@@ -778,6 +778,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.kv_caches = [
|
||||
cache for cache in kv_caches_dict.values() if cache.device == self.device
|
||||
]
|
||||
self.model_state.initialize_kv_cache(self.kv_cache_config, self.block_tables)
|
||||
if is_profiling:
|
||||
self.kv_connector = NO_OP_KV_CONNECTOR
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user