[Model Runner V2][Spec Decode] Add KV cache support for multi-layer MTP (#50062)
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
71b0da7c4e
commit
f80b66f548
@@ -5,6 +5,7 @@ from __future__ import annotations
|
||||
|
||||
import copy
|
||||
from collections import Counter
|
||||
from collections.abc import Collection
|
||||
from dataclasses import dataclass, fields, replace
|
||||
from enum import Enum, IntEnum
|
||||
from math import prod
|
||||
@@ -88,14 +89,24 @@ def is_quantized_kv_cache(kv_cache_dtype: str) -> bool:
|
||||
return get_kv_quant_mode(kv_cache_dtype) != KVQuantMode.NONE
|
||||
|
||||
|
||||
def replace_as(spec: KVCacheSpec, target_cls: type[_SpecT], **changes) -> _SpecT:
|
||||
def replace_as(
|
||||
spec: KVCacheSpec,
|
||||
target_cls: type[_SpecT],
|
||||
*,
|
||||
drop: Collection[str] = (),
|
||||
**changes,
|
||||
) -> _SpecT:
|
||||
"""``dataclasses.replace``, but rebuilding *spec* as *target_cls*
|
||||
e.g. ``SlidingWindowSpec`` -> ``FullAttentionSpec``
|
||||
|
||||
Every field of *spec* must exist on *target_cls*; fields only *target_cls* has keep
|
||||
their default values.
|
||||
Every field of *spec* must exist on *target_cls* unless named in *drop*;
|
||||
fields only *target_cls* has keep their default values.
|
||||
"""
|
||||
kwargs = {f.name: getattr(spec, f.name) for f in fields(spec) if f.init}
|
||||
kwargs = {
|
||||
f.name: getattr(spec, f.name)
|
||||
for f in fields(spec)
|
||||
if f.init and f.name not in drop
|
||||
}
|
||||
kwargs.update(changes)
|
||||
return target_cls(**kwargs)
|
||||
|
||||
@@ -573,6 +584,12 @@ class ChunkedLocalAttentionSpec(AttentionSpec):
|
||||
class SlidingWindowSpec(AttentionSpec):
|
||||
sliding_window: int
|
||||
head_size_v: int = None # type: ignore[assignment]
|
||||
# The trailing edge of the window is extended by ``extra_retained_tokens``
|
||||
# so that those extra trailing tokens' blocks are retained (but not
|
||||
# attended). This is needed for multi-module spec decoding which can
|
||||
# re-prefill the last num_spec_prefill_tokens - 1 tokens from the end
|
||||
# of the sequence, and thus needs to delay freeing/caching of blocks.
|
||||
extra_retained_tokens: int = 0
|
||||
|
||||
def __post_init__(self):
|
||||
if self.head_size_v is None:
|
||||
@@ -614,8 +631,13 @@ class SlidingWindowSpec(AttentionSpec):
|
||||
"""
|
||||
# During chunked prefill, we hold KV for the last `sliding_window-1`
|
||||
# computed tokens plus the in-flight tokens (frees happen on the
|
||||
# processed-token basis); never more than `max_model_len`.
|
||||
num_tokens = min(self.sliding_window - 1 + max_in_flight_tokens, max_model_len)
|
||||
# processed-token basis); never more than `max_model_len`. An additional
|
||||
# `extra_retained_tokens` trailing tokens are kept alive below the
|
||||
# window for multi-module spec decoding, and must be accounted here too.
|
||||
num_tokens = min(
|
||||
self.sliding_window - 1 + self.extra_retained_tokens + max_in_flight_tokens,
|
||||
max_model_len,
|
||||
)
|
||||
# +1 because the sliding window may not start from the beginning of
|
||||
# the block. E.g. block size 4 and num_token 4 needs two blocks
|
||||
# [XXCD][EF] to store the 6-token window [CDEF].
|
||||
@@ -686,16 +708,18 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
model_version_set = set(spec.model_version for spec in specs)
|
||||
sliding_window_set = set(spec.sliding_window for spec in specs)
|
||||
block_stride_set = set(spec.indexes_kv_by_block_stride for spec in specs)
|
||||
extra_retained_set = set(spec.extra_retained_tokens for spec in specs)
|
||||
assert (
|
||||
len(cache_dtype_str_set) == 1
|
||||
and len(compress_ratio_set) == 1
|
||||
and len(model_version_set) == 1
|
||||
and len(sliding_window_set) == 1
|
||||
and len(block_stride_set) == 1
|
||||
and len(extra_retained_set) == 1
|
||||
), (
|
||||
"All attention layers in the same KV cache group must use the same "
|
||||
"quantization method, compress ratio, model version, sliding "
|
||||
"window size, and KV block stride indexing."
|
||||
"window size, KV block stride indexing, and retained token count."
|
||||
)
|
||||
return cls(
|
||||
block_size=specs[0].block_size,
|
||||
@@ -705,6 +729,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
page_size_padded=specs[0].page_size_padded,
|
||||
indexes_kv_by_block_stride=block_stride_set.pop(),
|
||||
sliding_window=sliding_window_set.pop(),
|
||||
extra_retained_tokens=extra_retained_set.pop(),
|
||||
cache_dtype_str=cache_dtype_str_set.pop(),
|
||||
compress_ratio=compress_ratio_set.pop(),
|
||||
model_version=model_version_set.pop(),
|
||||
|
||||
Reference in New Issue
Block a user