[4/N][KV-Cache Layout Refactor] Promote local KV cache specs via a class-changing replace helper (#51612)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Lucas Wilkinson
2026-08-11 17:29:29 -04:00
committed by GitHub
co-authored by Claude
parent cb30f6f8dd
commit 61874f9842
2 changed files with 35 additions and 36 deletions
+20 -35
View File
@@ -33,6 +33,7 @@ from vllm.v1.kv_cache_interface import (
SlidingWindowMLASpec, SlidingWindowMLASpec,
SlidingWindowSpec, SlidingWindowSpec,
UniformTypeKVCacheSpecs, UniformTypeKVCacheSpecs,
replace_as,
) )
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
from vllm.v1.request import Request from vllm.v1.request import Request
@@ -1449,43 +1450,27 @@ def _promote_local_kv_cache_specs(
) )
return max(spec.page_size_padded, unpadded_page_size) return max(spec.page_size_padded, unpadded_page_size)
promotions: dict[type[AttentionSpec], type[AttentionSpec]] = {
SlidingWindowMLASpec: MLAAttentionSpec,
SlidingWindowSpec: FullAttentionSpec,
ChunkedLocalAttentionSpec: FullAttentionSpec,
}
if has_full_attention and (has_sliding_window or has_chunked_local_attention): if has_full_attention and (has_sliding_window or has_chunked_local_attention):
for layer_name, spec in kv_cache_spec.items(): for layer_name, spec in kv_cache_spec.items():
if isinstance(spec, SlidingWindowMLASpec): target_cls = next(
block_size = full_attention_block_size or spec.block_size (promotions[c] for c in type(spec).__mro__ if c in promotions), None
promoted_specs[layer_name] = MLAAttentionSpec( )
block_size=block_size, if target_cls is None:
num_kv_heads=spec.num_kv_heads, continue
head_size=spec.head_size, assert isinstance(spec, AttentionSpec)
dtype=spec.dtype, block_size = full_attention_block_size or spec.block_size
page_size_padded=promoted_page_size_padded(spec, block_size), promoted_specs[layer_name] = replace_as(
cache_dtype_str=spec.cache_dtype_str, spec,
alignment=spec.alignment, target_cls,
compress_ratio=spec.compress_ratio, block_size=block_size,
model_version=spec.model_version, page_size_padded=promoted_page_size_padded(spec, block_size),
) )
elif isinstance(spec, SlidingWindowSpec):
block_size = full_attention_block_size or spec.block_size
promoted_specs[layer_name] = FullAttentionSpec(
block_size=block_size,
num_kv_heads=spec.num_kv_heads,
head_size=spec.head_size,
head_size_v=spec.head_size_v,
dtype=spec.dtype,
kv_quant_mode=spec.kv_quant_mode,
sliding_window=spec.sliding_window,
page_size_padded=promoted_page_size_padded(spec, block_size),
)
elif isinstance(spec, ChunkedLocalAttentionSpec):
block_size = full_attention_block_size or spec.block_size
promoted_specs[layer_name] = FullAttentionSpec(
block_size=block_size,
num_kv_heads=spec.num_kv_heads,
head_size=spec.head_size,
dtype=spec.dtype,
attention_chunk_size=spec.attention_chunk_size,
page_size_padded=promoted_page_size_padded(spec, block_size),
)
if not ( if not (
is_kv_cache_spec_uniform(promoted_specs) is_kv_cache_spec_uniform(promoted_specs)
+15 -1
View File
@@ -8,7 +8,7 @@ from collections import Counter
from dataclasses import dataclass, fields, replace from dataclasses import dataclass, fields, replace
from enum import Enum, IntEnum from enum import Enum, IntEnum
from math import prod from math import prod
from typing import TYPE_CHECKING from typing import TYPE_CHECKING, TypeVar
import torch import torch
from typing_extensions import Self from typing_extensions import Self
@@ -24,6 +24,8 @@ if TYPE_CHECKING:
logger = init_logger(__name__) logger = init_logger(__name__)
_SpecT = TypeVar("_SpecT", bound="KVCacheSpec")
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# KV cache quantization mode # KV cache quantization mode
@@ -86,6 +88,18 @@ def is_quantized_kv_cache(kv_cache_dtype: str) -> bool:
return get_kv_quant_mode(kv_cache_dtype) != KVQuantMode.NONE return get_kv_quant_mode(kv_cache_dtype) != KVQuantMode.NONE
def replace_as(spec: KVCacheSpec, target_cls: type[_SpecT], **changes) -> _SpecT:
"""``dataclasses.replace``, but rebuilding *spec* as *target_cls*
e.g. ``SlidingWindowSpec`` -> ``FullAttentionSpec``
Every field of *spec* must exist on *target_cls*; fields only *target_cls* has keep
their default values.
"""
kwargs = {f.name: getattr(spec, f.name) for f in fields(spec) if f.init}
kwargs.update(changes)
return target_cls(**kwargs)
def kv_cache_uses_per_token_head_scales(kv_cache_dtype: str) -> bool: def kv_cache_uses_per_token_head_scales(kv_cache_dtype: str) -> bool:
"""Return True if *kv_cache_dtype* needs per-token-head scales.""" """Return True if *kv_cache_dtype* needs per-token-head scales."""
return get_kv_quant_mode(kv_cache_dtype).is_per_token_head return get_kv_quant_mode(kv_cache_dtype).is_per_token_head