[Model] Support Unlimited OCR (#46564)

Signed-off-by: Tianyu Guo <guoty9@mail2.sysu.edu.cn>
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
Tianyu Guo
2026-06-27 23:09:18 -07:00
committed by GitHub
co-authored by Isotr0py Roger Wang
parent a65f93fb2e
commit c6741b2ad4
35 changed files with 1084 additions and 49 deletions
@@ -73,6 +73,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
"RadioConfig": "vllm.transformers_utils.configs.radio",
"SpeculatorsConfig": "vllm.transformers_utils.configs.speculators",
"UltravoxConfig": "vllm.transformers_utils.configs.ultravox",
"UnlimitedOCRConfig": "vllm.transformers_utils.configs.unlimited_ocr",
"Step3VLConfig": "vllm.transformers_utils.configs.step3_vl",
"Step3VisionEncoderConfig": "vllm.transformers_utils.configs.step3_vl",
"Step3TextConfig": "vllm.transformers_utils.configs.step3_vl",
@@ -147,6 +148,7 @@ __all__ = [
"RadioConfig",
"SpeculatorsConfig",
"UltravoxConfig",
"UnlimitedOCRConfig",
"Step3VLConfig",
"Step3VisionEncoderConfig",
"Step3TextConfig",
@@ -0,0 +1,35 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Unlimited-OCR (baidu/Unlimited-OCR) reuses
# the DeepSeek-OCR multimodal layout (DeepEncoder = SAM-ViT-B + CLIP-L, a linear
# MLP projector and a DeepSeek-V2 text backbone). The only architectural
# difference is the language model, which is a DeepSeek-V2 *MoE* with plain
# multi-head attention (``use_mla=False``) instead of the dense MLA backbone.
# We therefore reuse ``DeepseekVLV2Config`` for parsing the nested config.
from vllm.transformers_utils.configs.deepseek_vl2 import DeepseekVLV2Config
class UnlimitedOCRConfig(DeepseekVLV2Config):
model_type = "unlimited-ocr"
# An explicit ``__init__`` is required: Transformers v5 processes each
# concrete config class' ``__init__`` signature to build nested sub-configs,
# and an empty subclass (only overriding ``model_type``) would skip
# ``DeepseekVLV2Config.__init__``, leaving ``text_config`` unset.
def __init__(
self,
tile_tag: str = "2D",
global_view_pos: str = "head",
candidate_resolutions: tuple[tuple[int, int]] = ((384, 384),),
rswa_window: int = 128,
**kwargs,
):
super().__init__(
tile_tag=tile_tag,
global_view_pos=global_view_pos,
candidate_resolutions=candidate_resolutions,
**kwargs,
)
self.rswa_window = rswa_window