[4/N][KV-Cache Layout Refactor] Promote local KV cache specs via a class-changing replace helper (#51612)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude
parent
cb30f6f8dd
commit
61874f9842
@@ -8,7 +8,7 @@ from collections import Counter
|
||||
from dataclasses import dataclass, fields, replace
|
||||
from enum import Enum, IntEnum
|
||||
from math import prod
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, TypeVar
|
||||
|
||||
import torch
|
||||
from typing_extensions import Self
|
||||
@@ -24,6 +24,8 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_SpecT = TypeVar("_SpecT", bound="KVCacheSpec")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# KV cache quantization mode
|
||||
@@ -86,6 +88,18 @@ def is_quantized_kv_cache(kv_cache_dtype: str) -> bool:
|
||||
return get_kv_quant_mode(kv_cache_dtype) != KVQuantMode.NONE
|
||||
|
||||
|
||||
def replace_as(spec: KVCacheSpec, target_cls: type[_SpecT], **changes) -> _SpecT:
|
||||
"""``dataclasses.replace``, but rebuilding *spec* as *target_cls*
|
||||
e.g. ``SlidingWindowSpec`` -> ``FullAttentionSpec``
|
||||
|
||||
Every field of *spec* must exist on *target_cls*; fields only *target_cls* has keep
|
||||
their default values.
|
||||
"""
|
||||
kwargs = {f.name: getattr(spec, f.name) for f in fields(spec) if f.init}
|
||||
kwargs.update(changes)
|
||||
return target_cls(**kwargs)
|
||||
|
||||
|
||||
def kv_cache_uses_per_token_head_scales(kv_cache_dtype: str) -> bool:
|
||||
"""Return True if *kv_cache_dtype* needs per-token-head scales."""
|
||||
return get_kv_quant_mode(kv_cache_dtype).is_per_token_head
|
||||
|
||||
Reference in New Issue
Block a user