[Model] Support Unlimited OCR (#46564)

Signed-off-by: Tianyu Guo <guoty9@mail2.sysu.edu.cn>
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
Tianyu Guo
2026-06-27 23:09:18 -07:00
committed by GitHub
co-authored by Isotr0py Roger Wang
parent a65f93fb2e
commit c6741b2ad4
35 changed files with 1084 additions and 49 deletions
+40
View File
@@ -437,6 +437,46 @@ class HiddenStateCacheSpec(MLAAttentionSpec):
pass
@dataclass(frozen=True, kw_only=True)
class RSWASpec(FullAttentionSpec):
"""KV cache spec for Reference Sliding Window Attention (R-SWA).
Prefill (image + text prompt) tokens are always globally visible.
Only the last ``rswa_window`` generated tokens are kept in the KV cache;
gap blocks (between the prefill tail and the current decode window) are
evicted during each decode step to bound memory at
O(prefix_blocks + window_blocks).
"""
rswa_window: int
@classmethod
def merge(cls, specs: list[RSWASpec]) -> RSWASpec:
assert all(isinstance(spec, RSWASpec) for spec in specs), (
"All attention layers in the same KV cache group must be RSWASpec."
)
rswa_windows = {spec.rswa_window for spec in specs}
assert len(rswa_windows) == 1, (
f"All R-SWA layers must share the same rswa_window, got {rswa_windows}"
)
# Delegate common field merging to the parent, then reattach rswa_window.
base = FullAttentionSpec.merge(specs) # type: ignore[arg-type]
return cls(
block_size=base.block_size,
num_kv_heads=base.num_kv_heads,
head_size=base.head_size,
head_size_v=base.head_size_v,
dtype=base.dtype,
kv_quant_mode=base.kv_quant_mode,
page_size_padded=base.page_size_padded,
indexes_kv_by_block_stride=base.indexes_kv_by_block_stride,
sliding_window=base.sliding_window,
attention_chunk_size=base.attention_chunk_size,
non_causal=base.non_causal,
rswa_window=rswa_windows.pop(),
)
@dataclass(frozen=True, kw_only=True)
class ChunkedLocalAttentionSpec(AttentionSpec):
attention_chunk_size: int