[Model Runner V2] Support sharing kv cache layers (#35045)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-05-22 22:18:23 +00:00
committed by GitHub
parent e203006a8b
commit 47d4407d7c
4 changed files with 54 additions and 19 deletions
+3
View File
@@ -441,6 +441,9 @@ def add_kv_sharing_layers_to_kv_cache_groups(
from the KV cache of `shared_kv_cache_layers[layer_name]`.
kv_cache_groups: The KV cache groups of the model.
"""
if not shared_kv_cache_layers:
return
layer_to_kv_cache_group: dict[str, KVCacheGroupSpec] = {}
for kv_cache_group in kv_cache_groups:
for layer_name in kv_cache_group.layer_names: