[6/N][KV-Cache Layout Refactor] Standardize KV cache layout (#51718)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
Lucas Wilkinson
2026-08-21 17:47:49 -07:00
committed by GitHub
parent 9ff7041b55
commit 8bdc70ec7b
150 changed files with 4006 additions and 5815 deletions
+8
View File
@@ -151,6 +151,14 @@ class Executor(ABC):
def get_kv_cache_specs(self) -> list[dict[str, KVCacheSpec]]:
return self.collective_rpc("get_kv_cache_spec")
def get_supported_kv_cache_layouts(self) -> list[list[str]]:
"""Layouts each worker's backends support, most preferred first."""
return self.collective_rpc("get_supported_kv_cache_layouts")
def set_kv_cache_layout(self, layout_name: str) -> None:
"""Publish the resolved KV cache layout to the workers."""
self.collective_rpc("set_kv_cache_layout", args=(layout_name,))
@overload
def collective_rpc(
self,