[4/N][KV-Cache Layout Refactor] Promote local KV cache specs via a class-changing replace helper (#51612)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude
parent
cb30f6f8dd
commit
61874f9842
@@ -33,6 +33,7 @@ from vllm.v1.kv_cache_interface import (
|
|||||||
SlidingWindowMLASpec,
|
SlidingWindowMLASpec,
|
||||||
SlidingWindowSpec,
|
SlidingWindowSpec,
|
||||||
UniformTypeKVCacheSpecs,
|
UniformTypeKVCacheSpecs,
|
||||||
|
replace_as,
|
||||||
)
|
)
|
||||||
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
|
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
|
||||||
from vllm.v1.request import Request
|
from vllm.v1.request import Request
|
||||||
@@ -1449,43 +1450,27 @@ def _promote_local_kv_cache_specs(
|
|||||||
)
|
)
|
||||||
return max(spec.page_size_padded, unpadded_page_size)
|
return max(spec.page_size_padded, unpadded_page_size)
|
||||||
|
|
||||||
|
promotions: dict[type[AttentionSpec], type[AttentionSpec]] = {
|
||||||
|
SlidingWindowMLASpec: MLAAttentionSpec,
|
||||||
|
SlidingWindowSpec: FullAttentionSpec,
|
||||||
|
ChunkedLocalAttentionSpec: FullAttentionSpec,
|
||||||
|
}
|
||||||
|
|
||||||
if has_full_attention and (has_sliding_window or has_chunked_local_attention):
|
if has_full_attention and (has_sliding_window or has_chunked_local_attention):
|
||||||
for layer_name, spec in kv_cache_spec.items():
|
for layer_name, spec in kv_cache_spec.items():
|
||||||
if isinstance(spec, SlidingWindowMLASpec):
|
target_cls = next(
|
||||||
block_size = full_attention_block_size or spec.block_size
|
(promotions[c] for c in type(spec).__mro__ if c in promotions), None
|
||||||
promoted_specs[layer_name] = MLAAttentionSpec(
|
)
|
||||||
block_size=block_size,
|
if target_cls is None:
|
||||||
num_kv_heads=spec.num_kv_heads,
|
continue
|
||||||
head_size=spec.head_size,
|
assert isinstance(spec, AttentionSpec)
|
||||||
dtype=spec.dtype,
|
block_size = full_attention_block_size or spec.block_size
|
||||||
page_size_padded=promoted_page_size_padded(spec, block_size),
|
promoted_specs[layer_name] = replace_as(
|
||||||
cache_dtype_str=spec.cache_dtype_str,
|
spec,
|
||||||
alignment=spec.alignment,
|
target_cls,
|
||||||
compress_ratio=spec.compress_ratio,
|
block_size=block_size,
|
||||||
model_version=spec.model_version,
|
page_size_padded=promoted_page_size_padded(spec, block_size),
|
||||||
)
|
)
|
||||||
elif isinstance(spec, SlidingWindowSpec):
|
|
||||||
block_size = full_attention_block_size or spec.block_size
|
|
||||||
promoted_specs[layer_name] = FullAttentionSpec(
|
|
||||||
block_size=block_size,
|
|
||||||
num_kv_heads=spec.num_kv_heads,
|
|
||||||
head_size=spec.head_size,
|
|
||||||
head_size_v=spec.head_size_v,
|
|
||||||
dtype=spec.dtype,
|
|
||||||
kv_quant_mode=spec.kv_quant_mode,
|
|
||||||
sliding_window=spec.sliding_window,
|
|
||||||
page_size_padded=promoted_page_size_padded(spec, block_size),
|
|
||||||
)
|
|
||||||
elif isinstance(spec, ChunkedLocalAttentionSpec):
|
|
||||||
block_size = full_attention_block_size or spec.block_size
|
|
||||||
promoted_specs[layer_name] = FullAttentionSpec(
|
|
||||||
block_size=block_size,
|
|
||||||
num_kv_heads=spec.num_kv_heads,
|
|
||||||
head_size=spec.head_size,
|
|
||||||
dtype=spec.dtype,
|
|
||||||
attention_chunk_size=spec.attention_chunk_size,
|
|
||||||
page_size_padded=promoted_page_size_padded(spec, block_size),
|
|
||||||
)
|
|
||||||
|
|
||||||
if not (
|
if not (
|
||||||
is_kv_cache_spec_uniform(promoted_specs)
|
is_kv_cache_spec_uniform(promoted_specs)
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ from collections import Counter
|
|||||||
from dataclasses import dataclass, fields, replace
|
from dataclasses import dataclass, fields, replace
|
||||||
from enum import Enum, IntEnum
|
from enum import Enum, IntEnum
|
||||||
from math import prod
|
from math import prod
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING, TypeVar
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
from typing_extensions import Self
|
from typing_extensions import Self
|
||||||
@@ -24,6 +24,8 @@ if TYPE_CHECKING:
|
|||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
_SpecT = TypeVar("_SpecT", bound="KVCacheSpec")
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# KV cache quantization mode
|
# KV cache quantization mode
|
||||||
@@ -86,6 +88,18 @@ def is_quantized_kv_cache(kv_cache_dtype: str) -> bool:
|
|||||||
return get_kv_quant_mode(kv_cache_dtype) != KVQuantMode.NONE
|
return get_kv_quant_mode(kv_cache_dtype) != KVQuantMode.NONE
|
||||||
|
|
||||||
|
|
||||||
|
def replace_as(spec: KVCacheSpec, target_cls: type[_SpecT], **changes) -> _SpecT:
|
||||||
|
"""``dataclasses.replace``, but rebuilding *spec* as *target_cls*
|
||||||
|
e.g. ``SlidingWindowSpec`` -> ``FullAttentionSpec``
|
||||||
|
|
||||||
|
Every field of *spec* must exist on *target_cls*; fields only *target_cls* has keep
|
||||||
|
their default values.
|
||||||
|
"""
|
||||||
|
kwargs = {f.name: getattr(spec, f.name) for f in fields(spec) if f.init}
|
||||||
|
kwargs.update(changes)
|
||||||
|
return target_cls(**kwargs)
|
||||||
|
|
||||||
|
|
||||||
def kv_cache_uses_per_token_head_scales(kv_cache_dtype: str) -> bool:
|
def kv_cache_uses_per_token_head_scales(kv_cache_dtype: str) -> bool:
|
||||||
"""Return True if *kv_cache_dtype* needs per-token-head scales."""
|
"""Return True if *kv_cache_dtype* needs per-token-head scales."""
|
||||||
return get_kv_quant_mode(kv_cache_dtype).is_per_token_head
|
return get_kv_quant_mode(kv_cache_dtype).is_per_token_head
|
||||||
|
|||||||
Reference in New Issue
Block a user