[Model] Support Unlimited OCR (#46564)
Signed-off-by: Tianyu Guo <guoty9@mail2.sysu.edu.cn> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
co-authored by
Isotr0py
Roger Wang
parent
a65f93fb2e
commit
c6741b2ad4
@@ -437,6 +437,46 @@ class HiddenStateCacheSpec(MLAAttentionSpec):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class RSWASpec(FullAttentionSpec):
|
||||
"""KV cache spec for Reference Sliding Window Attention (R-SWA).
|
||||
|
||||
Prefill (image + text prompt) tokens are always globally visible.
|
||||
Only the last ``rswa_window`` generated tokens are kept in the KV cache;
|
||||
gap blocks (between the prefill tail and the current decode window) are
|
||||
evicted during each decode step to bound memory at
|
||||
O(prefix_blocks + window_blocks).
|
||||
"""
|
||||
|
||||
rswa_window: int
|
||||
|
||||
@classmethod
|
||||
def merge(cls, specs: list[RSWASpec]) -> RSWASpec:
|
||||
assert all(isinstance(spec, RSWASpec) for spec in specs), (
|
||||
"All attention layers in the same KV cache group must be RSWASpec."
|
||||
)
|
||||
rswa_windows = {spec.rswa_window for spec in specs}
|
||||
assert len(rswa_windows) == 1, (
|
||||
f"All R-SWA layers must share the same rswa_window, got {rswa_windows}"
|
||||
)
|
||||
# Delegate common field merging to the parent, then reattach rswa_window.
|
||||
base = FullAttentionSpec.merge(specs) # type: ignore[arg-type]
|
||||
return cls(
|
||||
block_size=base.block_size,
|
||||
num_kv_heads=base.num_kv_heads,
|
||||
head_size=base.head_size,
|
||||
head_size_v=base.head_size_v,
|
||||
dtype=base.dtype,
|
||||
kv_quant_mode=base.kv_quant_mode,
|
||||
page_size_padded=base.page_size_padded,
|
||||
indexes_kv_by_block_stride=base.indexes_kv_by_block_stride,
|
||||
sliding_window=base.sliding_window,
|
||||
attention_chunk_size=base.attention_chunk_size,
|
||||
non_causal=base.non_causal,
|
||||
rswa_window=rswa_windows.pop(),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class ChunkedLocalAttentionSpec(AttentionSpec):
|
||||
attention_chunk_size: int
|
||||
|
||||
Reference in New Issue
Block a user