Merge branch 'main' into patches
This commit is contained in:
@@ -64,6 +64,7 @@ if TYPE_CHECKING:
|
||||
from vllm.tasks import ScoreType
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, KVCacheSpec
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import (
|
||||
EncoderCudaGraphCaptureInputs,
|
||||
EncoderCudaGraphConfig,
|
||||
@@ -670,6 +671,47 @@ def supports_score_template(
|
||||
return getattr(model, "supports_score_template", False)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsKVCacheGrouping(Protocol):
|
||||
"""The interface required for models that own their KV cache grouping.
|
||||
|
||||
.. note::
|
||||
Experimental. The classmethod contract may change as model-owned
|
||||
grouping stabilizes.
|
||||
|
||||
Implementing it lets a model replace the default KV cache grouping (which
|
||||
layers share a block table) while Core keeps owning placement, byte
|
||||
accounting, and admission.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def get_kv_cache_groups(
|
||||
cls,
|
||||
vllm_config: "VllmConfig",
|
||||
kv_cache_spec: dict[str, "KVCacheSpec"],
|
||||
) -> list["KVCacheGroupSpec"]:
|
||||
"""Split the model's layers into logical KV cache groups."""
|
||||
...
|
||||
|
||||
|
||||
@overload
|
||||
def supports_kv_cache_grouping(
|
||||
model: type[object],
|
||||
) -> TypeIs[type[SupportsKVCacheGrouping]]: ...
|
||||
|
||||
|
||||
@overload
|
||||
def supports_kv_cache_grouping(
|
||||
model: object,
|
||||
) -> TypeIs[SupportsKVCacheGrouping]: ...
|
||||
|
||||
|
||||
def supports_kv_cache_grouping(
|
||||
model: type[object] | object,
|
||||
) -> TypeIs[type[SupportsKVCacheGrouping]] | TypeIs[SupportsKVCacheGrouping]:
|
||||
return isinstance(model, SupportsKVCacheGrouping)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsLoRA(Protocol):
|
||||
"""The interface required for all models that support LoRA."""
|
||||
|
||||
Reference in New Issue
Block a user