[Model] Support Unlimited OCR (#46564)
Signed-off-by: Tianyu Guo <guoty9@mail2.sysu.edu.cn> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
co-authored by
Isotr0py
Roger Wang
parent
a65f93fb2e
commit
c6741b2ad4
@@ -29,6 +29,7 @@ _MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK: dict[str, ChatTemplatePath] = {
|
||||
"colpali": CHAT_TEMPLATES_DIR / "template_basic.jinja",
|
||||
"deepseek_ocr": CHAT_TEMPLATES_DIR / "template_deepseek_ocr.jinja",
|
||||
"deepseek_ocr2": CHAT_TEMPLATES_DIR / "template_deepseek_ocr.jinja",
|
||||
"unlimited-ocr": CHAT_TEMPLATES_DIR / "template_deepseek_ocr.jinja",
|
||||
"deepseek_vl_v2": CHAT_TEMPLATES_DIR / "template_deepseek_vl2.jinja",
|
||||
"fuyu": CHAT_TEMPLATES_DIR / "template_fuyu.jinja",
|
||||
"minicpmv": _get_minicpmv_chat_template_fallback,
|
||||
|
||||
@@ -124,6 +124,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
|
||||
laguna="LagunaConfig",
|
||||
lfm2_moe="Lfm2MoeConfig",
|
||||
tarsier2="Tarsier2Config",
|
||||
**{"unlimited-ocr": "UnlimitedOCRConfig"},
|
||||
)
|
||||
|
||||
_SPECULATIVE_DECODING_CONFIGS: set[str] = {"eagle", "speculators"}
|
||||
|
||||
@@ -73,6 +73,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
|
||||
"RadioConfig": "vllm.transformers_utils.configs.radio",
|
||||
"SpeculatorsConfig": "vllm.transformers_utils.configs.speculators",
|
||||
"UltravoxConfig": "vllm.transformers_utils.configs.ultravox",
|
||||
"UnlimitedOCRConfig": "vllm.transformers_utils.configs.unlimited_ocr",
|
||||
"Step3VLConfig": "vllm.transformers_utils.configs.step3_vl",
|
||||
"Step3VisionEncoderConfig": "vllm.transformers_utils.configs.step3_vl",
|
||||
"Step3TextConfig": "vllm.transformers_utils.configs.step3_vl",
|
||||
@@ -147,6 +148,7 @@ __all__ = [
|
||||
"RadioConfig",
|
||||
"SpeculatorsConfig",
|
||||
"UltravoxConfig",
|
||||
"UnlimitedOCRConfig",
|
||||
"Step3VLConfig",
|
||||
"Step3VisionEncoderConfig",
|
||||
"Step3TextConfig",
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Unlimited-OCR (baidu/Unlimited-OCR) reuses
|
||||
# the DeepSeek-OCR multimodal layout (DeepEncoder = SAM-ViT-B + CLIP-L, a linear
|
||||
# MLP projector and a DeepSeek-V2 text backbone). The only architectural
|
||||
# difference is the language model, which is a DeepSeek-V2 *MoE* with plain
|
||||
# multi-head attention (``use_mla=False``) instead of the dense MLA backbone.
|
||||
# We therefore reuse ``DeepseekVLV2Config`` for parsing the nested config.
|
||||
|
||||
from vllm.transformers_utils.configs.deepseek_vl2 import DeepseekVLV2Config
|
||||
|
||||
|
||||
class UnlimitedOCRConfig(DeepseekVLV2Config):
|
||||
model_type = "unlimited-ocr"
|
||||
|
||||
# An explicit ``__init__`` is required: Transformers v5 processes each
|
||||
# concrete config class' ``__init__`` signature to build nested sub-configs,
|
||||
# and an empty subclass (only overriding ``model_type``) would skip
|
||||
# ``DeepseekVLV2Config.__init__``, leaving ``text_config`` unset.
|
||||
def __init__(
|
||||
self,
|
||||
tile_tag: str = "2D",
|
||||
global_view_pos: str = "head",
|
||||
candidate_resolutions: tuple[tuple[int, int]] = ((384, 384),),
|
||||
rswa_window: int = 128,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(
|
||||
tile_tag=tile_tag,
|
||||
global_view_pos=global_view_pos,
|
||||
candidate_resolutions=candidate_resolutions,
|
||||
**kwargs,
|
||||
)
|
||||
self.rswa_window = rswa_window
|
||||
@@ -310,6 +310,12 @@ class ModelArchConfigConvertorBase:
|
||||
return False
|
||||
return self.hf_config.model_type in MM_PREFIX_LM_MODELS
|
||||
|
||||
def rswa_window(self) -> int | None:
|
||||
value = getattr(self.hf_config, "rswa_window", None)
|
||||
if value is None:
|
||||
return None
|
||||
return int(value)
|
||||
|
||||
def derive_max_model_len_and_key(self) -> tuple[float, str | None]:
|
||||
derived_max_model_len = float("inf")
|
||||
possible_keys = [
|
||||
@@ -360,6 +366,7 @@ class ModelArchConfigConvertorBase:
|
||||
quantization_config=self.get_quantization_config(),
|
||||
is_deepseek_mla=self.is_deepseek_mla(),
|
||||
is_mm_prefix_lm=self.is_mm_prefix_lm(),
|
||||
rswa_window=self.rswa_window(),
|
||||
derived_max_model_len_and_key=self.derive_max_model_len_and_key(),
|
||||
)
|
||||
|
||||
|
||||
@@ -161,10 +161,12 @@ class DeepseekOCRProcessor(ProcessorMixin):
|
||||
image_size: int = IMAGE_SIZE,
|
||||
base_size: int = BASE_SIZE,
|
||||
strategy: Literal["v1", "v2"] = "v1",
|
||||
max_crops: int = MAX_CROPS,
|
||||
**kwargs,
|
||||
):
|
||||
self.image_size = image_size
|
||||
self.base_size = base_size
|
||||
self.max_crops = max_crops
|
||||
|
||||
# image token calculation strategy for
|
||||
# Deepseek-OCR and Deepseek-OCR-2
|
||||
@@ -332,7 +334,7 @@ class DeepseekOCRProcessor(ProcessorMixin):
|
||||
crop_ratio = [1, 1]
|
||||
elif cropping:
|
||||
images_crop_raw, crop_ratio = dynamic_preprocess(
|
||||
image, image_size=self.image_size
|
||||
image, image_size=self.image_size, max_num=self.max_crops
|
||||
)
|
||||
else:
|
||||
crop_ratio = [1, 1]
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Image processor for Unlimited-OCR (baidu/Unlimited-OCR)."""
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.transformers_utils.processors.deepseek_ocr import DeepseekOCRProcessor
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class UnlimitedOCRProcessor(DeepseekOCRProcessor):
|
||||
"""DeepseekOCRProcessor variant for Unlimited-OCR.
|
||||
|
||||
The only behavioural difference from the base processor is a multi-image
|
||||
safeguard: when more than one image is present, crop ("gundam") mode is
|
||||
disabled.
|
||||
|
||||
Because the effective crop flag then depends on *how many* images are in the
|
||||
request, the per-item processing output is no longer invariant of sibling
|
||||
images. ``UnlimitedOCRMultiModalProcessor`` accounts for this by bypassing
|
||||
the multimodal processing cache for multi-image requests (see its
|
||||
``_cached_apply_hf_processor``), so the two paths stay consistent.
|
||||
|
||||
DeepSeek-OCR does *not* have this restriction because its ``max_crops=6`` is
|
||||
small enough to be safe for multi-image use.
|
||||
"""
|
||||
|
||||
def tokenize_with_images(
|
||||
self,
|
||||
conversation: str,
|
||||
images: list[Image.Image],
|
||||
bos: bool = True,
|
||||
eos: bool = True,
|
||||
cropping: bool = True,
|
||||
):
|
||||
if len(images) > 1 and cropping:
|
||||
logger.warning_once(
|
||||
"Unlimited-OCR: crop mode is not supported for multi-image "
|
||||
"input. Falling back to cropping=False."
|
||||
)
|
||||
cropping = False
|
||||
return super().tokenize_with_images(
|
||||
conversation, images, bos=bos, eos=eos, cropping=cropping
|
||||
)
|
||||
Reference in New Issue
Block a user