feat(distributed): add get_required_kvcache_layout class method to kv connector api (#20433)

Signed-off-by: wxsm <wxsms@foxmail.com>
This commit is contained in:
wxsm
2025-07-30 16:41:51 +00:00
committed by GitHub
parent 4904e53c32
commit f4135232b9
7 changed files with 186 additions and 28 deletions
@@ -9,7 +9,7 @@ The class provides two primary abstract methods:
"""
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Union
from typing import TYPE_CHECKING, Optional, Union
import torch
@@ -124,5 +124,19 @@ class KVConnectorBase(ABC):
raise NotImplementedError
@classmethod
def get_required_kvcache_layout(
cls, vllm_config: "VllmConfig") -> Optional[str]:
"""
Get the required KV cache layout for this connector.
Args:
vllm_config (VllmConfig): the vllm config.
Returns:
str: the required KV cache layout. e.g. HND, or NHD.
None if the connector does not require a specific layout.
"""
return None
KVConnectorBaseType = Union[KVConnectorBase, KVConnectorBase_V1]