[4/N][KV-Cache Layout Refactor] Promote local KV cache specs via a class-changing replace helper (#51612)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude
parent
cb30f6f8dd
commit
61874f9842
@@ -33,6 +33,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
SlidingWindowMLASpec,
|
||||
SlidingWindowSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
replace_as,
|
||||
)
|
||||
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
|
||||
from vllm.v1.request import Request
|
||||
@@ -1449,43 +1450,27 @@ def _promote_local_kv_cache_specs(
|
||||
)
|
||||
return max(spec.page_size_padded, unpadded_page_size)
|
||||
|
||||
promotions: dict[type[AttentionSpec], type[AttentionSpec]] = {
|
||||
SlidingWindowMLASpec: MLAAttentionSpec,
|
||||
SlidingWindowSpec: FullAttentionSpec,
|
||||
ChunkedLocalAttentionSpec: FullAttentionSpec,
|
||||
}
|
||||
|
||||
if has_full_attention and (has_sliding_window or has_chunked_local_attention):
|
||||
for layer_name, spec in kv_cache_spec.items():
|
||||
if isinstance(spec, SlidingWindowMLASpec):
|
||||
block_size = full_attention_block_size or spec.block_size
|
||||
promoted_specs[layer_name] = MLAAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=spec.num_kv_heads,
|
||||
head_size=spec.head_size,
|
||||
dtype=spec.dtype,
|
||||
page_size_padded=promoted_page_size_padded(spec, block_size),
|
||||
cache_dtype_str=spec.cache_dtype_str,
|
||||
alignment=spec.alignment,
|
||||
compress_ratio=spec.compress_ratio,
|
||||
model_version=spec.model_version,
|
||||
)
|
||||
elif isinstance(spec, SlidingWindowSpec):
|
||||
block_size = full_attention_block_size or spec.block_size
|
||||
promoted_specs[layer_name] = FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=spec.num_kv_heads,
|
||||
head_size=spec.head_size,
|
||||
head_size_v=spec.head_size_v,
|
||||
dtype=spec.dtype,
|
||||
kv_quant_mode=spec.kv_quant_mode,
|
||||
sliding_window=spec.sliding_window,
|
||||
page_size_padded=promoted_page_size_padded(spec, block_size),
|
||||
)
|
||||
elif isinstance(spec, ChunkedLocalAttentionSpec):
|
||||
block_size = full_attention_block_size or spec.block_size
|
||||
promoted_specs[layer_name] = FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=spec.num_kv_heads,
|
||||
head_size=spec.head_size,
|
||||
dtype=spec.dtype,
|
||||
attention_chunk_size=spec.attention_chunk_size,
|
||||
page_size_padded=promoted_page_size_padded(spec, block_size),
|
||||
)
|
||||
target_cls = next(
|
||||
(promotions[c] for c in type(spec).__mro__ if c in promotions), None
|
||||
)
|
||||
if target_cls is None:
|
||||
continue
|
||||
assert isinstance(spec, AttentionSpec)
|
||||
block_size = full_attention_block_size or spec.block_size
|
||||
promoted_specs[layer_name] = replace_as(
|
||||
spec,
|
||||
target_cls,
|
||||
block_size=block_size,
|
||||
page_size_padded=promoted_page_size_padded(spec, block_size),
|
||||
)
|
||||
|
||||
if not (
|
||||
is_kv_cache_spec_uniform(promoted_specs)
|
||||
|
||||
@@ -8,7 +8,7 @@ from collections import Counter
|
||||
from dataclasses import dataclass, fields, replace
|
||||
from enum import Enum, IntEnum
|
||||
from math import prod
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, TypeVar
|
||||
|
||||
import torch
|
||||
from typing_extensions import Self
|
||||
@@ -24,6 +24,8 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_SpecT = TypeVar("_SpecT", bound="KVCacheSpec")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# KV cache quantization mode
|
||||
@@ -86,6 +88,18 @@ def is_quantized_kv_cache(kv_cache_dtype: str) -> bool:
|
||||
return get_kv_quant_mode(kv_cache_dtype) != KVQuantMode.NONE
|
||||
|
||||
|
||||
def replace_as(spec: KVCacheSpec, target_cls: type[_SpecT], **changes) -> _SpecT:
|
||||
"""``dataclasses.replace``, but rebuilding *spec* as *target_cls*
|
||||
e.g. ``SlidingWindowSpec`` -> ``FullAttentionSpec``
|
||||
|
||||
Every field of *spec* must exist on *target_cls*; fields only *target_cls* has keep
|
||||
their default values.
|
||||
"""
|
||||
kwargs = {f.name: getattr(spec, f.name) for f in fields(spec) if f.init}
|
||||
kwargs.update(changes)
|
||||
return target_cls(**kwargs)
|
||||
|
||||
|
||||
def kv_cache_uses_per_token_head_scales(kv_cache_dtype: str) -> bool:
|
||||
"""Return True if *kv_cache_dtype* needs per-token-head scales."""
|
||||
return get_kv_quant_mode(kv_cache_dtype).is_per_token_head
|
||||
|
||||
Reference in New Issue
Block a user