[Model] Support Unlimited OCR (#46564)
Signed-off-by: Tianyu Guo <guoty9@mail2.sysu.edu.cn> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
co-authored by
Isotr0py
Roger Wang
parent
a65f93fb2e
commit
c6741b2ad4
@@ -2335,6 +2335,13 @@ class GPUModelRunner(
|
||||
req_idx = self.input_batch.req_id_to_index[req_id]
|
||||
req_doc_ranges[req_idx] = image_doc_ranges
|
||||
|
||||
# Reference Sliding Window Attention (R-SWA): pass per-request prompt
|
||||
# lengths so the attention backend can keep the prefix globally visible.
|
||||
# The backend owns the persistent CUDA-graph-safe GPU buffer.
|
||||
rswa_prefix_lens = None
|
||||
if self.model_config.rswa_window is not None:
|
||||
rswa_prefix_lens = num_prompt_tokens_cpu
|
||||
|
||||
cm_base = CommonAttentionMetadata(
|
||||
query_start_loc=self.query_start_loc.gpu[: num_reqs_padded + 1],
|
||||
query_start_loc_cpu=self.query_start_loc.cpu[: num_reqs_padded + 1],
|
||||
@@ -2352,6 +2359,7 @@ class GPUModelRunner(
|
||||
is_prefilling=is_prefilling,
|
||||
positions=self.positions[:num_tokens_padded],
|
||||
mm_req_doc_ranges=req_doc_ranges,
|
||||
rswa_prefix_lens=rswa_prefix_lens,
|
||||
)
|
||||
|
||||
if self.dcp_world_size > 1:
|
||||
|
||||
Reference in New Issue
Block a user