[Model] Support Unlimited OCR (#46564)
Signed-off-by: Tianyu Guo <guoty9@mail2.sysu.edu.cn> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
co-authored by
Isotr0py
Roger Wang
parent
a65f93fb2e
commit
c6741b2ad4
@@ -14,6 +14,7 @@ from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderA
|
||||
from vllm.model_executor.layers.attention.prefill_prefix_lm_attention import (
|
||||
PrefillPrefixLMAttention,
|
||||
)
|
||||
from vllm.model_executor.layers.attention.rswa_attention import RSWAAttention
|
||||
from vllm.model_executor.layers.attention.static_sink_attention import (
|
||||
StaticSinkAttention,
|
||||
)
|
||||
@@ -26,5 +27,6 @@ __all__ = [
|
||||
"MLAAttention",
|
||||
"MMEncoderAttention",
|
||||
"PrefillPrefixLMAttention",
|
||||
"RSWAAttention",
|
||||
"StaticSinkAttention",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.config.vllm import VllmConfig
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.v1.kv_cache_interface import KVCacheSpec, RSWASpec, get_kv_quant_mode
|
||||
|
||||
|
||||
class RSWAAttention(Attention):
|
||||
"""Attention layer that reports ``RSWASpec`` as its KV cache spec.
|
||||
|
||||
Drop-in replacement for the standard ``Attention`` layer when the model is
|
||||
configured with Reference Sliding Window Attention (R-SWA,
|
||||
``rswa_window > 0``). The actual masking logic lives in the attention
|
||||
backend (FlexAttention or FA4 mask_mod); this layer only overrides
|
||||
``get_kv_cache_spec`` so the KV cache manager instantiates ``RSWAManager``
|
||||
(instead of ``FullAttentionManager``) and can therefore evict "gap" blocks
|
||||
to keep per-request KV memory bounded at O(prefix + window).
|
||||
"""
|
||||
|
||||
def __init__(self, *args, rswa_window: int, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self._rswa_window = rswa_window
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None:
|
||||
spec = super().get_kv_cache_spec(vllm_config)
|
||||
if spec is None:
|
||||
return None
|
||||
return RSWASpec(
|
||||
block_size=vllm_config.cache_config.block_size,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_size=self.head_size,
|
||||
head_size_v=self.head_size_v,
|
||||
dtype=self.kv_cache_torch_dtype,
|
||||
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
|
||||
rswa_window=self._rswa_window,
|
||||
)
|
||||
@@ -54,6 +54,139 @@ class Gemma3TextModelConfig(VerifyAndUpdateConfig):
|
||||
hf_config.is_causal = not hf_config.use_bidirectional_attention
|
||||
|
||||
|
||||
class UnlimitedOCRForCausalLMConfig(VerifyAndUpdateConfig):
|
||||
@staticmethod
|
||||
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
|
||||
"""Configure Unlimited-OCR attention backends for R-SWA and vision.
|
||||
|
||||
Backend selection — controlled by the standard ``--attention-config``
|
||||
CLI argument (priority order):
|
||||
|
||||
1. ``--attention-config '{"backend": "FLASH_ATTN"}'``
|
||||
→ FA4 + rswa_mask_mod. Exact token-level R-SWA.
|
||||
``flash_attn_version`` is forced to 4 if not already set (R-SWA
|
||||
mask_mod requires FA4; FA3 cannot express it). Raises if FA4 is
|
||||
not available on this device.
|
||||
|
||||
2. ``--attention-config '{"backend": "FLEX_ATTENTION"}'``
|
||||
→ FlexAttention R-SWA via Triton block mask.
|
||||
|
||||
3. ``--attention-config '{"backend": "auto"}'`` (or omitted)
|
||||
→ Auto-detect: FA4 if available (H20/H100 SM90), else FlexAttention.
|
||||
|
||||
Regardless of backend, prefix caching is disabled for this model: R-SWA
|
||||
decode-phase KV is not a pure causal function of the prefix (so decode
|
||||
blocks are not reusable), and single-turn image-led OCR prompts rarely
|
||||
hit the prefix cache.
|
||||
|
||||
Example — force FlexAttention even on a machine with FA4::
|
||||
|
||||
vllm serve baidu/Unlimited-OCR \\
|
||||
--attention-config '{"backend": "FLEX_ATTENTION"}'
|
||||
"""
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.vllm_flash_attn import is_fa_version_supported
|
||||
|
||||
attn_config = vllm_config.attention_config
|
||||
fa4_available = is_fa_version_supported(4)
|
||||
|
||||
# ── step 1: resolve backend ─────────────────────────────────────────
|
||||
# None means the user did not explicitly specify a backend; auto-select.
|
||||
if attn_config.backend is None:
|
||||
attn_config.backend = (
|
||||
AttentionBackendEnum.FLASH_ATTN
|
||||
if fa4_available
|
||||
else AttentionBackendEnum.FLEX_ATTENTION
|
||||
)
|
||||
logger.info(
|
||||
"Unlimited-OCR: auto-selected attention backend=%s (fa4_available=%s).",
|
||||
attn_config.backend.value,
|
||||
fa4_available,
|
||||
)
|
||||
|
||||
# ── step 2: configure the chosen backend ────────────────────────────
|
||||
if attn_config.backend == AttentionBackendEnum.FLASH_ATTN:
|
||||
if not fa4_available:
|
||||
raise RuntimeError(
|
||||
"Unlimited-OCR: --attention-config backend=FLASH_ATTN "
|
||||
"requires FA4 (rswa_mask_mod), but FA4 is not available on "
|
||||
"this device/installation. Use backend=FLEX_ATTENTION or "
|
||||
"upgrade vllm-flash-attn."
|
||||
)
|
||||
# On SM90 (H20), the default FA version is FA3 regardless of FA4
|
||||
# availability (FA4 is only auto-upgraded when head_size > 256).
|
||||
# The R-SWA mask_mod requires FA4, so force the version globally.
|
||||
if attn_config.flash_attn_version is None:
|
||||
attn_config.flash_attn_version = 4
|
||||
elif attn_config.flash_attn_version < 4:
|
||||
logger.warning(
|
||||
"Unlimited-OCR: flash_attn_version=%d cannot express the "
|
||||
"R-SWA mask_mod; upgrading to 4.",
|
||||
attn_config.flash_attn_version,
|
||||
)
|
||||
attn_config.flash_attn_version = 4
|
||||
logger.info(
|
||||
"Unlimited-OCR: FlashAttention FA%d + rswa_mask_mod — exact R-SWA.",
|
||||
attn_config.flash_attn_version,
|
||||
)
|
||||
|
||||
elif attn_config.backend == AttentionBackendEnum.FLEX_ATTENTION:
|
||||
logger.info(
|
||||
"Unlimited-OCR: FlexAttention — R-SWA via Triton block mask%s.",
|
||||
""
|
||||
if not fa4_available
|
||||
else (
|
||||
" (FA4 available but not used; pass backend=FLASH_ATTN to upgrade)"
|
||||
),
|
||||
)
|
||||
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unlimited-OCR: unsupported attention backend "
|
||||
f"{attn_config.backend!r} for R-SWA. "
|
||||
"Use FLASH_ATTN (FA4) or FLEX_ATTENTION."
|
||||
)
|
||||
|
||||
# R-SWA windows the *generated* tokens, so a decode-token's KV is not a
|
||||
# pure causal function of the prefix and cannot be safely reused across
|
||||
# requests via prefix caching. Only the prompt/image prefix is cacheable,
|
||||
# but OCR is single-turn with image-led prompts that rarely share a
|
||||
# prefix, so prefix caching brings little benefit while complicating the
|
||||
# KV cache manager. Disable it for this model.
|
||||
cache_config = vllm_config.cache_config
|
||||
if cache_config.enable_prefix_caching:
|
||||
cache_config.enable_prefix_caching = False
|
||||
logger.info(
|
||||
"Unlimited-OCR: disabling prefix caching (R-SWA decode KV is not "
|
||||
"cacheable, and single-turn image-led prompts rarely hit the "
|
||||
"prefix cache)."
|
||||
)
|
||||
|
||||
mm_config = getattr(vllm_config.model_config, "multimodal_config", None)
|
||||
if mm_config is not None:
|
||||
if mm_config.mm_encoder_attn_backend is None:
|
||||
mm_config.mm_encoder_attn_backend = AttentionBackendEnum.FLASH_ATTN
|
||||
elif mm_config.mm_encoder_attn_backend == AttentionBackendEnum.FLASHINFER:
|
||||
logger.warning(
|
||||
"Unlimited-OCR: FlashInfer is not supported for the vision "
|
||||
"encoder (the CLIP stage runs full attention without "
|
||||
"cu_seqlens); falling back to FlashAttention."
|
||||
)
|
||||
mm_config.mm_encoder_attn_backend = AttentionBackendEnum.FLASH_ATTN
|
||||
|
||||
@staticmethod
|
||||
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
|
||||
text_config = model_config.hf_config.text_config
|
||||
text_config.architectures = ["DeepseekV2ForCausalLM"]
|
||||
if getattr(model_config.hf_config, "rswa_window", None) is None:
|
||||
model_config.hf_config.rswa_window = 128
|
||||
# Propagate rswa_window to text_config so that DeepseekAttention (which
|
||||
# receives text_config as its vllm_config.model_config.hf_config via
|
||||
# init_vllm_registered_model) can read it and create RSWAAttention.
|
||||
rswa_window = model_config.hf_config.rswa_window
|
||||
text_config.rswa_window = rswa_window
|
||||
|
||||
|
||||
class Gemma4Config(VerifyAndUpdateConfig):
|
||||
@staticmethod
|
||||
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
|
||||
@@ -703,6 +836,7 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
|
||||
"Qwen3VLForSequenceClassification": Qwen3VLForSequenceClassificationConfig,
|
||||
"Qwen3_5ForConditionalGeneration": Qwen3_5ForConditionalGenerationConfig,
|
||||
"Qwen3_5MoeForConditionalGeneration": Qwen3_5ForConditionalGenerationConfig,
|
||||
"UnlimitedOCRForCausalLM": UnlimitedOCRForCausalLMConfig,
|
||||
"VoyageQwen3BidirectionalEmbedModel": VoyageQwen3BidirectionalEmbedModelConfig,
|
||||
"XLMRobertaModel": JinaRobertaModelConfig,
|
||||
}
|
||||
|
||||
@@ -220,8 +220,10 @@ class DeepseekOCRProcessingInfo(BaseProcessingInfo):
|
||||
patch_size = 16
|
||||
downsample_ratio = 4
|
||||
|
||||
if CROP_MODE:
|
||||
if image_width <= 640 and image_height <= 640:
|
||||
# Use the caller-supplied `cropping` flag so that callers that disable
|
||||
# crop mode for multi-image requests get a consistent token count.
|
||||
if cropping:
|
||||
if image_width <= IMAGE_SIZE and image_height <= IMAGE_SIZE:
|
||||
crop_ratio = [1, 1]
|
||||
else:
|
||||
# find the closest aspect ratio to the target
|
||||
|
||||
@@ -45,7 +45,7 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.attention import Attention, RSWAAttention
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
@@ -174,15 +174,28 @@ class DeepseekAttention(nn.Module):
|
||||
max_position=max_position_embeddings,
|
||||
rope_parameters=config.rope_parameters,
|
||||
)
|
||||
self.attn = Attention(
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
self.scaling,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.attn",
|
||||
)
|
||||
rswa_window = getattr(vllm_config.model_config.hf_config, "rswa_window", None)
|
||||
if rswa_window is not None:
|
||||
self.attn = RSWAAttention(
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
self.scaling,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.attn",
|
||||
rswa_window=rswa_window,
|
||||
)
|
||||
else:
|
||||
self.attn = Attention(
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
self.scaling,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.attn",
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -588,12 +601,12 @@ class DeepseekV32IndexerCache(torch.nn.Module, AttentionLayerBase):
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
|
||||
return MLAAttentionSpec( # Only has one vector instead of K + V
|
||||
return MLAAttentionSpec(
|
||||
block_size=self.cache_config.block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=self.head_dim,
|
||||
dtype=self.dtype,
|
||||
)
|
||||
) # Only has one vector instead of K + V
|
||||
|
||||
def forward(self): ...
|
||||
|
||||
|
||||
@@ -358,6 +358,7 @@ _MULTIMODAL_MODELS = {
|
||||
"DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
|
||||
"DeepseekOCRForCausalLM": ("deepseek_ocr", "DeepseekOCRForCausalLM"),
|
||||
"DeepseekOCR2ForCausalLM": ("deepseek_ocr2", "DeepseekOCR2ForCausalLM"),
|
||||
"UnlimitedOCRForCausalLM": ("unlimited_ocr", "UnlimitedOCRForCausalLM"),
|
||||
"DotsOCRForCausalLM": ("dots_ocr", "DotsOCRForCausalLM"),
|
||||
"Eagle2_5_VLForConditionalGeneration": (
|
||||
"eagle2_5_vl",
|
||||
|
||||
@@ -0,0 +1,250 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inference-only Unlimited-OCR model compatible with HuggingFace weights.
|
||||
|
||||
Unlimited-OCR (``baidu/Unlimited-OCR``) shares
|
||||
the exact DeepSeek-OCR (gundam, ``base_size=1024`` / ``image_size=640`` / crop)
|
||||
vision stack: a DeepEncoder (SAM-ViT-B + CLIP-L) followed by a linear MLP
|
||||
projector, with the same image-token tiling layout. The only difference is the
|
||||
language backbone, which is a DeepSeek-V2 *MoE* (64 routed + 2 shared experts,
|
||||
``first_k_dense_replace=1``) that uses plain multi-head attention
|
||||
(``use_mla=False``, ``qk_nope_head_dim == qk_rope_head_dim == 0``) instead of
|
||||
the dense MLA decoder used by DeepSeek-OCR.
|
||||
|
||||
vLLM's ``DeepseekV2DecoderLayer`` already dispatches to the plain-MHA
|
||||
``DeepseekAttention`` whenever ``qk_nope_head_dim == qk_rope_head_dim == 0`` and
|
||||
builds the MoE blocks straight from the config, so the whole DeepSeek-OCR
|
||||
multimodal wrapper can be reused verbatim. Model-specific config (language
|
||||
backbone architecture, FlexAttention for R-SWA, vision encoder backend, and
|
||||
``rswa_window``) is applied in ``UnlimitedOCRForCausalLMConfig``.
|
||||
|
||||
Attention backend: the reference applies Reference Sliding Window Attention
|
||||
(R-SWA) -- the prompt/image tokens form a globally-visible prefix while the
|
||||
*generated* tokens additionally attend only a fixed sliding window (128) of
|
||||
recent tokens. We reproduce this (Level 1: full KV cache + custom mask) by
|
||||
forcing the language model onto the FlexAttention backend and installing an
|
||||
R-SWA ``mask_mod``. FlexAttention is the only backend able to express the
|
||||
"global prefix + sliding window" mask; FlashAttention-3 / Triton only support a
|
||||
uniform window (and additionally crash or compute incorrectly on this decoder's
|
||||
10-head,
|
||||
head_dim-128 shape), and FlashInfer's paged decode exposes no custom mask. The
|
||||
window size is published via ``model_config.rswa_window``, which the model
|
||||
runner reads to plumb per-request prefix lengths into the FlexAttention mask.
|
||||
|
||||
The *vision encoder* (DeepEncoder's CLIP stage, head_dim 64) is unaffected and
|
||||
does not use R-SWA: it runs a single full-attention prefill pass. FlashAttention,
|
||||
Triton and torch SDPA all produce correct, equally fast results; only FlashInfer
|
||||
is incompatible (its ViT path asserts on the varlen cu_seqlens metadata that this
|
||||
CLIP encoder never builds). We default the encoder to FlashAttention and
|
||||
transparently fall back to it if FlashInfer is requested.
|
||||
|
||||
To suppress repetition on long documents, use ``NGramPerReqLogitsProcessor`` from
|
||||
this module (same request-level processor as DeepSeek-OCR) with::
|
||||
|
||||
SamplingParams(
|
||||
temperature=0.0,
|
||||
max_tokens=8192,
|
||||
extra_args={"ngram_size": 35, "window_size": 128},
|
||||
)
|
||||
|
||||
Image processing
|
||||
----------------
|
||||
Unlimited-OCR supports up to 32 local crops (vs 6 for DeepSeek-OCR), i.e.
|
||||
``dynamic_preprocess`` runs with ``max_num=32``.
|
||||
|
||||
Multi-image requests fall back to non-crop mode: crop ("gundam") mode is only
|
||||
used for single-image input. DeepSeek-OCR does *not* have this restriction.
|
||||
|
||||
Because that fallback makes the per-image processor output depend on *how many*
|
||||
images are in the request, it breaks the assumption behind vLLM's per-item
|
||||
multimodal processing cache (``MultiModalProcessorOnlyCache``). We handle this
|
||||
the same way ``DeepseekVL2MultiModalProcessor`` does: only the single-image case
|
||||
(which always crops) is cached, while multi-image requests bypass the cache and
|
||||
are recomputed fresh -- see ``_cached_apply_hf_processor`` below. This keeps the
|
||||
processing cache consistent (verified by ``test_processing_correctness``).
|
||||
"""
|
||||
|
||||
import math
|
||||
from collections.abc import Mapping, Sequence
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalKwargsItems
|
||||
from vllm.multimodal.parse import (
|
||||
ImageEmbeddingItems,
|
||||
ImageProcessorItems,
|
||||
ImageSize,
|
||||
MultiModalDataItems,
|
||||
)
|
||||
from vllm.multimodal.processing import PromptReplacement, PromptUpdate
|
||||
from vllm.multimodal.processing.context import TimingContext
|
||||
from vllm.multimodal.processing.inputs import ProcessorInputs
|
||||
from vllm.multimodal.processing.processor import MultiModalProcessingInfo
|
||||
from vllm.transformers_utils.processors.deepseek_ocr import (
|
||||
BASE_SIZE,
|
||||
CROP_MODE,
|
||||
IMAGE_SIZE,
|
||||
count_tiles,
|
||||
)
|
||||
|
||||
from .deepseek_ocr import (
|
||||
DeepseekOCRDummyInputsBuilder,
|
||||
DeepseekOCRForCausalLM,
|
||||
DeepseekOCRMultiModalProcessor,
|
||||
DeepseekOCRProcessingInfo,
|
||||
NGramPerReqLogitsProcessor,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"NGramPerReqLogitsProcessor",
|
||||
"UnlimitedOCRForCausalLM",
|
||||
]
|
||||
|
||||
# Unlimited-OCR supports up to 32 local crops (vs 6 for DeepSeek-OCR).
|
||||
_UNLIMITED_OCR_MAX_CROPS = 32
|
||||
|
||||
|
||||
class UnlimitedOCRProcessingInfo(DeepseekOCRProcessingInfo):
|
||||
"""ProcessingInfo for Unlimited-OCR: same as DeepSeek-OCR but with
|
||||
max_crops=32 instead of 6. The higher crop count allows tiling very large
|
||||
document pages into up to 32 640×640 patches (dynamic_preprocess max_num=32).
|
||||
"""
|
||||
|
||||
def get_hf_config(self):
|
||||
from vllm.transformers_utils.configs.unlimited_ocr import UnlimitedOCRConfig
|
||||
|
||||
return self.ctx.get_hf_config(UnlimitedOCRConfig)
|
||||
|
||||
def get_hf_processor(self, **kwargs: object):
|
||||
from vllm.transformers_utils.processors.unlimited_ocr import (
|
||||
UnlimitedOCRProcessor,
|
||||
)
|
||||
|
||||
v1_processor_config = dict(
|
||||
image_size=IMAGE_SIZE,
|
||||
base_size=BASE_SIZE,
|
||||
crop_mode=CROP_MODE,
|
||||
strategy="v1",
|
||||
max_crops=_UNLIMITED_OCR_MAX_CROPS,
|
||||
)
|
||||
return self.ctx.get_hf_processor(
|
||||
UnlimitedOCRProcessor,
|
||||
**{**v1_processor_config, **kwargs},
|
||||
)
|
||||
|
||||
def get_num_image_tokens(
|
||||
self, *, image_width: int, image_height: int, cropping: bool = True
|
||||
) -> int:
|
||||
patch_size = 16
|
||||
downsample_ratio = 4
|
||||
|
||||
# Honour the caller-supplied `cropping` flag: multi-image callers pass
|
||||
# cropping=False to match UnlimitedOCRProcessor.tokenize_with_images.
|
||||
if cropping:
|
||||
if image_width <= IMAGE_SIZE and image_height <= IMAGE_SIZE:
|
||||
crop_ratio = [1, 1]
|
||||
else:
|
||||
crop_ratio = count_tiles(
|
||||
image_width,
|
||||
image_height,
|
||||
max_num=_UNLIMITED_OCR_MAX_CROPS,
|
||||
image_size=IMAGE_SIZE,
|
||||
)
|
||||
num_width_tiles, num_height_tiles = crop_ratio
|
||||
else:
|
||||
num_width_tiles = num_height_tiles = 1
|
||||
|
||||
h = w = math.ceil((BASE_SIZE // patch_size) / downsample_ratio)
|
||||
h2 = w2 = math.ceil((IMAGE_SIZE // patch_size) / downsample_ratio)
|
||||
|
||||
global_views_tokens = h * (w + 1)
|
||||
if num_width_tiles > 1 or num_height_tiles > 1:
|
||||
local_views_tokens = (num_height_tiles * h2) * (num_width_tiles * w2 + 1)
|
||||
else:
|
||||
local_views_tokens = 0
|
||||
|
||||
return global_views_tokens + local_views_tokens + 1
|
||||
|
||||
def get_image_size_with_most_features(self) -> ImageSize:
|
||||
# With max_crops=32, the widest possible grid is 4×8 (aspect ratio 1:2).
|
||||
# A 2560×5120 image (4×640 × 8×640) selects exactly 4×8=32 tiles and
|
||||
# produces the maximum token count.
|
||||
return ImageSize(width=640 * 4, height=640 * 8)
|
||||
|
||||
|
||||
class UnlimitedOCRMultiModalProcessor(DeepseekOCRMultiModalProcessor):
|
||||
"""Multimodal processor for Unlimited-OCR.
|
||||
|
||||
Disables crop mode for multi-image requests (to stay consistent with
|
||||
``UnlimitedOCRProcessor.tokenize_with_images``), and -- since that makes the
|
||||
per-image output depend on the request's image count -- bypasses the
|
||||
per-item processing cache for multi-image requests, exactly like
|
||||
``DeepseekVL2MultiModalProcessor``.
|
||||
|
||||
DeepSeek-OCR does *not* apply either of these.
|
||||
"""
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
|
||||
image_token_id = hf_processor.image_token_id
|
||||
assert isinstance(image_token_id, int)
|
||||
|
||||
def get_replacement_unlimited_ocr(item_idx: int):
|
||||
images = mm_items.get_items(
|
||||
"image", (ImageEmbeddingItems, ImageProcessorItems)
|
||||
)
|
||||
|
||||
if isinstance(images, ImageEmbeddingItems):
|
||||
num_image_tokens = images.get_feature_size(item_idx)
|
||||
else:
|
||||
size = images.get_image_size(item_idx)
|
||||
|
||||
# Disable crop mode for multi-image input.
|
||||
# UnlimitedOCRProcessor.tokenize_with_images applies the same
|
||||
# fallback, so both paths must agree on the effective crop flag.
|
||||
effective_cropping = CROP_MODE and len(images) == 1
|
||||
|
||||
num_image_tokens = self.info.get_num_image_tokens(
|
||||
image_width=size.width,
|
||||
image_height=size.height,
|
||||
cropping=effective_cropping,
|
||||
)
|
||||
return [image_token_id] * num_image_tokens
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=[image_token_id],
|
||||
replacement=get_replacement_unlimited_ocr,
|
||||
)
|
||||
]
|
||||
|
||||
def _cached_apply_hf_processor(
|
||||
self,
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
|
||||
# The processor logic differs for single-image (crop) vs multi-image
|
||||
# (no crop) requests. The processing cache assumes per-item output is
|
||||
# invariant of how many images are passed per prompt, so we only cache
|
||||
# the single-image case and recompute multi-image requests fresh.
|
||||
if inputs.mm_data_items.get_count("image", strict=False) > 1:
|
||||
return self._apply_hf_processor(inputs, timing_ctx)
|
||||
|
||||
return super()._cached_apply_hf_processor(inputs, timing_ctx)
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
UnlimitedOCRMultiModalProcessor,
|
||||
info=UnlimitedOCRProcessingInfo,
|
||||
dummy_inputs=DeepseekOCRDummyInputsBuilder,
|
||||
)
|
||||
class UnlimitedOCRForCausalLM(DeepseekOCRForCausalLM):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
Reference in New Issue
Block a user