[Model Runner V2] Support sharing kv cache layers (#35045)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-05-22 22:18:23 +00:00
committed by GitHub
parent e203006a8b
commit 47d4407d7c
4 changed files with 54 additions and 19 deletions
+2 -3
View File
@@ -432,9 +432,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if self.speculator is not None:
# HACK(woosuk)
self.speculator.set_attn(
self.model_state,
self.kv_cache_config,
self.block_tables,
self.model_state, self.kv_cache_config, self.block_tables
)
self.kv_caches: list[torch.Tensor] = []
@@ -446,6 +444,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.device,
self.cache_config.cache_dtype,
kernel_block_sizes,
self.vllm_config,
)
self.kv_connector = get_kv_connector(self.vllm_config, kv_caches_dict)