[Feat][Model] Support encoder-side SWA-bounded replay for DeepSeek-V4.1-Flash (#56227)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
Yifan Qiao
2026-09-18 15:01:24 -07:00
committed by GitHub
co-authored by Claude Fable 5.1 Woosuk Kwon
parent 6aab78f477
commit 1dc2d854c1
30 changed files with 1067 additions and 38 deletions
+23 -1
View File
@@ -164,6 +164,11 @@ class KVCacheSpec:
"""Whether this spec's group participates in prefix caching."""
return True
@property
def prefix_replay_tokens(self) -> int:
"""DeepSeek-V4.1 only: bounded replay. Currently only for DSV41 SWA."""
return 0
@property
def num_heads(self) -> int:
raise NotImplementedError
@@ -908,6 +913,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
# DeepseekV4-only: see MLAAttentionSpec.model_version.
alignment: int | None = None # Default to None for no padding.
model_version: str | None = None
bounded_replay: bool = False
# MLA stores a single latent vector per state; there is no separate V.
head_size_v: int = 0
@@ -919,6 +925,14 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
super().__post_init__()
_apply_alignment_padding(self)
@property
def prefix_cacheable(self) -> bool:
return not self.bounded_replay
@property
def prefix_replay_tokens(self) -> int:
return self.sliding_window if self.bounded_replay else 0
@classmethod
def merge(cls, specs: list[Self]) -> Self:
assert all(isinstance(spec, SlidingWindowMLASpec) for spec in specs), (
@@ -930,16 +944,18 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
model_version_set = set(spec.model_version for spec in specs)
sliding_window_set = set(spec.sliding_window for spec in specs)
extra_retained_set = set(spec.extra_retained_tokens for spec in specs)
bounded_replay_set = set(spec.bounded_replay for spec in specs)
assert (
len(cache_dtype_str_set) == 1
and len(tokens_per_state_set) == 1
and len(model_version_set) == 1
and len(sliding_window_set) == 1
and len(extra_retained_set) == 1
and len(bounded_replay_set) == 1
), (
"All attention layers in the same KV cache group must use the same "
"quantization method, tokens per state, model version, sliding "
"window size, and retained token count."
"window size, retained token count, and replay policy."
)
return cls(
block_size=specs[0].block_size,
@@ -954,6 +970,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
cache_dtype_str=cache_dtype_str_set.pop(),
tokens_per_state=tokens_per_state_set.pop(),
model_version=model_version_set.pop(),
bounded_replay=bounded_replay_set.pop(),
)
def is_uniform_with_collection(
@@ -962,6 +979,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
return all(
isinstance(spec, SlidingWindowMLASpec)
and spec.sliding_window == self.sliding_window
and spec.bounded_replay == self.bounded_replay
for spec in kv_cache_specs.values()
)
@@ -1195,6 +1213,10 @@ class UniformTypeKVCacheSpecs(KVCacheSpec):
def prefix_cacheable(self) -> bool:
return all(spec.prefix_cacheable for spec in self.kv_cache_specs.values())
@property
def prefix_replay_tokens(self) -> int:
return max(spec.prefix_replay_tokens for spec in self.kv_cache_specs.values())
@property
def first_spec(self) -> KVCacheSpec:
"""Return the first spec in the group."""