[6/N][KV-Cache Layout Refactor] Standardize KV cache layout (#51718)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
@@ -151,6 +151,14 @@ class Executor(ABC):
|
||||
def get_kv_cache_specs(self) -> list[dict[str, KVCacheSpec]]:
|
||||
return self.collective_rpc("get_kv_cache_spec")
|
||||
|
||||
def get_supported_kv_cache_layouts(self) -> list[list[str]]:
|
||||
"""Layouts each worker's backends support, most preferred first."""
|
||||
return self.collective_rpc("get_supported_kv_cache_layouts")
|
||||
|
||||
def set_kv_cache_layout(self, layout_name: str) -> None:
|
||||
"""Publish the resolved KV cache layout to the workers."""
|
||||
self.collective_rpc("set_kv_cache_layout", args=(layout_name,))
|
||||
|
||||
@overload
|
||||
def collective_rpc(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user