[Model Runner V2] Support sharing kv cache layers (#35045)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-05-22 22:18:23 +00:00
committed by GitHub
parent e203006a8b
commit 47d4407d7c
4 changed files with 54 additions and 19 deletions
+6 -7
View File
@@ -8,6 +8,7 @@ import os
import tempfile
import threading
import time
from collections.abc import Iterable
from contextlib import contextmanager
from dataclasses import is_dataclass
from datetime import datetime
@@ -2220,7 +2221,7 @@ T = TypeVar("T")
def get_layers_from_vllm_config(
vllm_config: VllmConfig,
layer_type: type[T],
layer_names: list[str] | None = None,
layer_names: Iterable[str] | None = None,
) -> dict[str, T]:
"""
Get layers from the vLLM config.
@@ -2231,14 +2232,12 @@ def get_layers_from_vllm_config(
layer_names: The names of the layers to get. If None, return all layers.
"""
if layer_names is None:
layer_names = list(vllm_config.compilation_config.static_forward_context.keys())
forward_context = vllm_config.compilation_config.static_forward_context
if layer_names is None:
layer_names = forward_context.keys()
return {
layer_name: forward_context[layer_name]
layer_name: layer
for layer_name in layer_names
if layer_name in forward_context
and isinstance(forward_context[layer_name], layer_type)
if isinstance(layer := forward_context.get(layer_name), layer_type)
}