[KVCache] Support Pluggable KVCacheSpec (#37505)
Signed-off-by: MengqingCao <cmq0113@163.com> Signed-off-by: Mengqing Cao <cmq0113@163.com> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
zjy0516
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
df7252c343
commit
0c6631f02a
@@ -17,6 +17,7 @@ from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv, round_up
|
||||
from vllm.utils.torch_utils import get_dtype_size, nvfp4_kv_cache_full_dim
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
@@ -139,6 +140,21 @@ class KVCacheSpec:
|
||||
)
|
||||
return copy.deepcopy(specs[0])
|
||||
|
||||
def is_uniform_with_collection(
|
||||
self, kv_cache_specs: dict[str, KVCacheSpec]
|
||||
) -> bool:
|
||||
"""
|
||||
Whether this KVCacheSpec is uniform with all specs of all layers.
|
||||
"""
|
||||
uniform_type_base_spec = KVCacheSpecRegistry.get_uniform_type_base_spec(self)
|
||||
assert uniform_type_base_spec is not None, (
|
||||
f"Unsupported KV cache spec type: {type(self)}. "
|
||||
"Please register it using @register_kv_cache_spec decorator."
|
||||
)
|
||||
return all(
|
||||
isinstance(spec, uniform_type_base_spec) for spec in kv_cache_specs.values()
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class AttentionSpec(KVCacheSpec):
|
||||
@@ -430,6 +446,15 @@ class ChunkedLocalAttentionSpec(AttentionSpec):
|
||||
)
|
||||
return max_blocks * self.page_size_bytes
|
||||
|
||||
def is_uniform_with_collection(
|
||||
self, kv_cache_specs: dict[str, KVCacheSpec]
|
||||
) -> bool:
|
||||
return all(
|
||||
isinstance(spec, ChunkedLocalAttentionSpec)
|
||||
and spec.attention_chunk_size == self.attention_chunk_size
|
||||
for spec in kv_cache_specs.values()
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class SlidingWindowSpec(AttentionSpec):
|
||||
@@ -493,6 +518,15 @@ class SlidingWindowSpec(AttentionSpec):
|
||||
)
|
||||
return max_blocks * self.page_size_bytes
|
||||
|
||||
def is_uniform_with_collection(
|
||||
self, kv_cache_specs: dict[str, KVCacheSpec]
|
||||
) -> bool:
|
||||
return all(
|
||||
isinstance(spec, SlidingWindowSpec)
|
||||
and spec.sliding_window == self.sliding_window
|
||||
for spec in kv_cache_specs.values()
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
@@ -558,6 +592,15 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
model_version=model_version_set.pop(),
|
||||
)
|
||||
|
||||
def is_uniform_with_collection(
|
||||
self, kv_cache_specs: dict[str, KVCacheSpec]
|
||||
) -> bool:
|
||||
return all(
|
||||
isinstance(spec, SlidingWindowMLASpec)
|
||||
and spec.sliding_window == self.sliding_window
|
||||
for spec in kv_cache_specs.values()
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MambaSpec(KVCacheSpec):
|
||||
@@ -590,6 +633,15 @@ class MambaSpec(KVCacheSpec):
|
||||
else:
|
||||
return self.page_size_bytes * (1 + self.num_speculative_blocks)
|
||||
|
||||
def is_uniform_with_collection(
|
||||
self, kv_cache_specs: dict[str, KVCacheSpec]
|
||||
) -> bool:
|
||||
return all(
|
||||
isinstance(spec, MambaSpec)
|
||||
and spec.num_speculative_blocks == self.num_speculative_blocks
|
||||
for spec in kv_cache_specs.values()
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EncoderOnlyAttentionSpec(AttentionSpec):
|
||||
@@ -689,53 +741,16 @@ class UniformTypeKVCacheSpecs(KVCacheSpec):
|
||||
def is_uniform_type(cls, kv_cache_specs: dict[str, KVCacheSpec]) -> bool:
|
||||
"""
|
||||
Whether all layers have the same type of KV cache spec.
|
||||
|
||||
Uses the registry to determine grouping base classes, so custom specs
|
||||
that inherit from FullAttentionSpec are treated as full attention.
|
||||
"""
|
||||
block_sizes = set(spec.block_size for spec in kv_cache_specs.values())
|
||||
if len(block_sizes) > 1:
|
||||
# Different block sizes, not uniform.
|
||||
return False
|
||||
one_spec = next(iter(kv_cache_specs.values()))
|
||||
# NOTE: Check subclasses before parent classes since isinstance()
|
||||
# returns True for subclasses.
|
||||
if isinstance(one_spec, SlidingWindowMLASpec):
|
||||
# SlidingWindowMLASpec is uniform if all specs are SlidingWindowMLASpec
|
||||
# with the same sliding_window size.
|
||||
return all(
|
||||
isinstance(spec, SlidingWindowMLASpec)
|
||||
and spec.sliding_window == one_spec.sliding_window
|
||||
for spec in kv_cache_specs.values()
|
||||
)
|
||||
elif isinstance(one_spec, FullAttentionSpec):
|
||||
return all(
|
||||
isinstance(spec, FullAttentionSpec) for spec in kv_cache_specs.values()
|
||||
)
|
||||
elif isinstance(one_spec, CrossAttentionSpec):
|
||||
return all(
|
||||
isinstance(spec, CrossAttentionSpec) for spec in kv_cache_specs.values()
|
||||
)
|
||||
elif isinstance(one_spec, SlidingWindowSpec):
|
||||
return all(
|
||||
isinstance(spec, SlidingWindowSpec)
|
||||
and spec.sliding_window == one_spec.sliding_window
|
||||
for spec in kv_cache_specs.values()
|
||||
)
|
||||
elif isinstance(one_spec, ChunkedLocalAttentionSpec):
|
||||
return all(
|
||||
isinstance(spec, ChunkedLocalAttentionSpec)
|
||||
and spec.attention_chunk_size == one_spec.attention_chunk_size
|
||||
for spec in kv_cache_specs.values()
|
||||
)
|
||||
elif isinstance(one_spec, MambaSpec):
|
||||
return all(
|
||||
isinstance(spec, MambaSpec)
|
||||
and spec.num_speculative_blocks == one_spec.num_speculative_blocks
|
||||
for spec in kv_cache_specs.values()
|
||||
)
|
||||
else:
|
||||
# NOTE(Chen): Please add new branches for new KV cache spec types.
|
||||
raise NotImplementedError(
|
||||
f"Unsupported KV cache spec type: {type(one_spec)}"
|
||||
)
|
||||
first_spec = next(iter(kv_cache_specs.values()))
|
||||
return first_spec.is_uniform_with_collection(kv_cache_specs)
|
||||
|
||||
@classmethod
|
||||
def from_specs(cls, kv_cache_specs: dict[str, KVCacheSpec]) -> Self | None:
|
||||
|
||||
Reference in New Issue
Block a user