[Model] Support Unlimited OCR (#46564)

Signed-off-by: Tianyu Guo <guoty9@mail2.sysu.edu.cn>
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
Tianyu Guo
2026-06-27 23:09:18 -07:00
committed by GitHub
co-authored by Isotr0py Roger Wang
parent a65f93fb2e
commit c6741b2ad4
35 changed files with 1084 additions and 49 deletions
+17
View File
@@ -223,6 +223,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
max_num_tokens=self.max_num_tokens,
device=self.device,
)
# R-SWA: persistent GPU buffer for per-request prefix lengths (CUDA-graph safe).
self.rswa_prefix_lens_buffer: torch.Tensor | None = None
if self.model_config.rswa_window is not None:
self.rswa_prefix_lens_buffer = torch.zeros(
self.max_num_reqs, dtype=torch.int32, device=self.device
)
if self.use_pp:
self.pp_handler = PPHandler(
@@ -985,6 +991,16 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if self.use_pp:
# max_seq_len is only consumed by the PP `compute_need_sampled_mask`
max_seq_len_np = self.req_states.max_seq_len[idx_mapping_np]
rswa_prefix_lens = None
if self.rswa_prefix_lens_buffer is not None:
rswa_prefix_lens = self.rswa_prefix_lens_buffer[:num_reqs_padded]
rswa_prefix_lens[:num_reqs] = self.req_states.prompt_len.gpu[
idx_mapping[:num_reqs]
]
if num_reqs_padded > num_reqs:
rswa_prefix_lens[num_reqs:].zero_()
return InputBatch(
req_ids=req_ids,
num_reqs=num_reqs,
@@ -1015,6 +1031,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
cu_num_logits=cu_num_logits,
cu_num_logits_np=cu_num_logits_np,
has_structured_output_reqs=scheduler_output.has_structured_output_requests,
rswa_prefix_lens=rswa_prefix_lens,
)
def prepare_attn(