[Feat][Model] Support encoder-side SWA-bounded replay for DeepSeek-V4.1-Flash (#56227)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
co-authored by
Claude Fable 5.1
Woosuk Kwon
parent
6aab78f477
commit
1dc2d854c1
@@ -164,6 +164,11 @@ class KVCacheSpec:
|
||||
"""Whether this spec's group participates in prefix caching."""
|
||||
return True
|
||||
|
||||
@property
|
||||
def prefix_replay_tokens(self) -> int:
|
||||
"""DeepSeek-V4.1 only: bounded replay. Currently only for DSV41 SWA."""
|
||||
return 0
|
||||
|
||||
@property
|
||||
def num_heads(self) -> int:
|
||||
raise NotImplementedError
|
||||
@@ -908,6 +913,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
# DeepseekV4-only: see MLAAttentionSpec.model_version.
|
||||
alignment: int | None = None # Default to None for no padding.
|
||||
model_version: str | None = None
|
||||
bounded_replay: bool = False
|
||||
|
||||
# MLA stores a single latent vector per state; there is no separate V.
|
||||
head_size_v: int = 0
|
||||
@@ -919,6 +925,14 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
super().__post_init__()
|
||||
_apply_alignment_padding(self)
|
||||
|
||||
@property
|
||||
def prefix_cacheable(self) -> bool:
|
||||
return not self.bounded_replay
|
||||
|
||||
@property
|
||||
def prefix_replay_tokens(self) -> int:
|
||||
return self.sliding_window if self.bounded_replay else 0
|
||||
|
||||
@classmethod
|
||||
def merge(cls, specs: list[Self]) -> Self:
|
||||
assert all(isinstance(spec, SlidingWindowMLASpec) for spec in specs), (
|
||||
@@ -930,16 +944,18 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
model_version_set = set(spec.model_version for spec in specs)
|
||||
sliding_window_set = set(spec.sliding_window for spec in specs)
|
||||
extra_retained_set = set(spec.extra_retained_tokens for spec in specs)
|
||||
bounded_replay_set = set(spec.bounded_replay for spec in specs)
|
||||
assert (
|
||||
len(cache_dtype_str_set) == 1
|
||||
and len(tokens_per_state_set) == 1
|
||||
and len(model_version_set) == 1
|
||||
and len(sliding_window_set) == 1
|
||||
and len(extra_retained_set) == 1
|
||||
and len(bounded_replay_set) == 1
|
||||
), (
|
||||
"All attention layers in the same KV cache group must use the same "
|
||||
"quantization method, tokens per state, model version, sliding "
|
||||
"window size, and retained token count."
|
||||
"window size, retained token count, and replay policy."
|
||||
)
|
||||
return cls(
|
||||
block_size=specs[0].block_size,
|
||||
@@ -954,6 +970,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
cache_dtype_str=cache_dtype_str_set.pop(),
|
||||
tokens_per_state=tokens_per_state_set.pop(),
|
||||
model_version=model_version_set.pop(),
|
||||
bounded_replay=bounded_replay_set.pop(),
|
||||
)
|
||||
|
||||
def is_uniform_with_collection(
|
||||
@@ -962,6 +979,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
return all(
|
||||
isinstance(spec, SlidingWindowMLASpec)
|
||||
and spec.sliding_window == self.sliding_window
|
||||
and spec.bounded_replay == self.bounded_replay
|
||||
for spec in kv_cache_specs.values()
|
||||
)
|
||||
|
||||
@@ -1195,6 +1213,10 @@ class UniformTypeKVCacheSpecs(KVCacheSpec):
|
||||
def prefix_cacheable(self) -> bool:
|
||||
return all(spec.prefix_cacheable for spec in self.kv_cache_specs.values())
|
||||
|
||||
@property
|
||||
def prefix_replay_tokens(self) -> int:
|
||||
return max(spec.prefix_replay_tokens for spec in self.kv_cache_specs.values())
|
||||
|
||||
@property
|
||||
def first_spec(self) -> KVCacheSpec:
|
||||
"""Return the first spec in the group."""
|
||||
|
||||
Reference in New Issue
Block a user