[KVCache] Support Pluggable KVCacheSpec (#37505)

Signed-off-by: MengqingCao <cmq0113@163.com>
Signed-off-by: Mengqing Cao <cmq0113@163.com>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Mengqing Cao
2026-06-03 09:05:16 -07:00
committed by GitHub
co-authored by zjy0516 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent df7252c343
commit 0c6631f02a
11 changed files with 698 additions and 59 deletions
+57 -42
View File
@@ -17,6 +17,7 @@ from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv, round_up
from vllm.utils.torch_utils import get_dtype_size, nvfp4_kv_cache_full_dim
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
if TYPE_CHECKING:
from vllm.config import VllmConfig
@@ -139,6 +140,21 @@ class KVCacheSpec:
)
return copy.deepcopy(specs[0])
def is_uniform_with_collection(
self, kv_cache_specs: dict[str, KVCacheSpec]
) -> bool:
"""
Whether this KVCacheSpec is uniform with all specs of all layers.
"""
uniform_type_base_spec = KVCacheSpecRegistry.get_uniform_type_base_spec(self)
assert uniform_type_base_spec is not None, (
f"Unsupported KV cache spec type: {type(self)}. "
"Please register it using @register_kv_cache_spec decorator."
)
return all(
isinstance(spec, uniform_type_base_spec) for spec in kv_cache_specs.values()
)
@dataclass(frozen=True, kw_only=True)
class AttentionSpec(KVCacheSpec):
@@ -430,6 +446,15 @@ class ChunkedLocalAttentionSpec(AttentionSpec):
)
return max_blocks * self.page_size_bytes
def is_uniform_with_collection(
self, kv_cache_specs: dict[str, KVCacheSpec]
) -> bool:
return all(
isinstance(spec, ChunkedLocalAttentionSpec)
and spec.attention_chunk_size == self.attention_chunk_size
for spec in kv_cache_specs.values()
)
@dataclass(frozen=True, kw_only=True)
class SlidingWindowSpec(AttentionSpec):
@@ -493,6 +518,15 @@ class SlidingWindowSpec(AttentionSpec):
)
return max_blocks * self.page_size_bytes
def is_uniform_with_collection(
self, kv_cache_specs: dict[str, KVCacheSpec]
) -> bool:
return all(
isinstance(spec, SlidingWindowSpec)
and spec.sliding_window == self.sliding_window
for spec in kv_cache_specs.values()
)
@dataclass(frozen=True, kw_only=True)
class SlidingWindowMLASpec(SlidingWindowSpec):
@@ -558,6 +592,15 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
model_version=model_version_set.pop(),
)
def is_uniform_with_collection(
self, kv_cache_specs: dict[str, KVCacheSpec]
) -> bool:
return all(
isinstance(spec, SlidingWindowMLASpec)
and spec.sliding_window == self.sliding_window
for spec in kv_cache_specs.values()
)
@dataclass(frozen=True)
class MambaSpec(KVCacheSpec):
@@ -590,6 +633,15 @@ class MambaSpec(KVCacheSpec):
else:
return self.page_size_bytes * (1 + self.num_speculative_blocks)
def is_uniform_with_collection(
self, kv_cache_specs: dict[str, KVCacheSpec]
) -> bool:
return all(
isinstance(spec, MambaSpec)
and spec.num_speculative_blocks == self.num_speculative_blocks
for spec in kv_cache_specs.values()
)
@dataclass(frozen=True)
class EncoderOnlyAttentionSpec(AttentionSpec):
@@ -689,53 +741,16 @@ class UniformTypeKVCacheSpecs(KVCacheSpec):
def is_uniform_type(cls, kv_cache_specs: dict[str, KVCacheSpec]) -> bool:
"""
Whether all layers have the same type of KV cache spec.
Uses the registry to determine grouping base classes, so custom specs
that inherit from FullAttentionSpec are treated as full attention.
"""
block_sizes = set(spec.block_size for spec in kv_cache_specs.values())
if len(block_sizes) > 1:
# Different block sizes, not uniform.
return False
one_spec = next(iter(kv_cache_specs.values()))
# NOTE: Check subclasses before parent classes since isinstance()
# returns True for subclasses.
if isinstance(one_spec, SlidingWindowMLASpec):
# SlidingWindowMLASpec is uniform if all specs are SlidingWindowMLASpec
# with the same sliding_window size.
return all(
isinstance(spec, SlidingWindowMLASpec)
and spec.sliding_window == one_spec.sliding_window
for spec in kv_cache_specs.values()
)
elif isinstance(one_spec, FullAttentionSpec):
return all(
isinstance(spec, FullAttentionSpec) for spec in kv_cache_specs.values()
)
elif isinstance(one_spec, CrossAttentionSpec):
return all(
isinstance(spec, CrossAttentionSpec) for spec in kv_cache_specs.values()
)
elif isinstance(one_spec, SlidingWindowSpec):
return all(
isinstance(spec, SlidingWindowSpec)
and spec.sliding_window == one_spec.sliding_window
for spec in kv_cache_specs.values()
)
elif isinstance(one_spec, ChunkedLocalAttentionSpec):
return all(
isinstance(spec, ChunkedLocalAttentionSpec)
and spec.attention_chunk_size == one_spec.attention_chunk_size
for spec in kv_cache_specs.values()
)
elif isinstance(one_spec, MambaSpec):
return all(
isinstance(spec, MambaSpec)
and spec.num_speculative_blocks == one_spec.num_speculative_blocks
for spec in kv_cache_specs.values()
)
else:
# NOTE(Chen): Please add new branches for new KV cache spec types.
raise NotImplementedError(
f"Unsupported KV cache spec type: {type(one_spec)}"
)
first_spec = next(iter(kv_cache_specs.values()))
return first_spec.is_uniform_with_collection(kv_cache_specs)
@classmethod
def from_specs(cls, kv_cache_specs: dict[str, KVCacheSpec]) -> Self | None: