[Model] Support Unlimited OCR (#46564)

Signed-off-by: Tianyu Guo <guoty9@mail2.sysu.edu.cn>
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
Tianyu Guo
2026-06-27 23:09:18 -07:00
committed by GitHub
co-authored by Isotr0py Roger Wang
parent a65f93fb2e
commit c6741b2ad4
35 changed files with 1084 additions and 49 deletions
@@ -14,6 +14,7 @@ from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderA
from vllm.model_executor.layers.attention.prefill_prefix_lm_attention import (
PrefillPrefixLMAttention,
)
from vllm.model_executor.layers.attention.rswa_attention import RSWAAttention
from vllm.model_executor.layers.attention.static_sink_attention import (
StaticSinkAttention,
)
@@ -26,5 +27,6 @@ __all__ = [
"MLAAttention",
"MMEncoderAttention",
"PrefillPrefixLMAttention",
"RSWAAttention",
"StaticSinkAttention",
]
@@ -0,0 +1,37 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.config.vllm import VllmConfig
from vllm.model_executor.layers.attention import Attention
from vllm.v1.kv_cache_interface import KVCacheSpec, RSWASpec, get_kv_quant_mode
class RSWAAttention(Attention):
"""Attention layer that reports ``RSWASpec`` as its KV cache spec.
Drop-in replacement for the standard ``Attention`` layer when the model is
configured with Reference Sliding Window Attention (R-SWA,
``rswa_window > 0``). The actual masking logic lives in the attention
backend (FlexAttention or FA4 mask_mod); this layer only overrides
``get_kv_cache_spec`` so the KV cache manager instantiates ``RSWAManager``
(instead of ``FullAttentionManager``) and can therefore evict "gap" blocks
to keep per-request KV memory bounded at O(prefix + window).
"""
def __init__(self, *args, rswa_window: int, **kwargs) -> None:
super().__init__(*args, **kwargs)
self._rswa_window = rswa_window
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None:
spec = super().get_kv_cache_spec(vllm_config)
if spec is None:
return None
return RSWASpec(
block_size=vllm_config.cache_config.block_size,
num_kv_heads=self.num_kv_heads,
head_size=self.head_size,
head_size_v=self.head_size_v,
dtype=self.kv_cache_torch_dtype,
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
rswa_window=self._rswa_window,
)
+134
View File
@@ -54,6 +54,139 @@ class Gemma3TextModelConfig(VerifyAndUpdateConfig):
hf_config.is_causal = not hf_config.use_bidirectional_attention
class UnlimitedOCRForCausalLMConfig(VerifyAndUpdateConfig):
@staticmethod
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
"""Configure Unlimited-OCR attention backends for R-SWA and vision.
Backend selection — controlled by the standard ``--attention-config``
CLI argument (priority order):
1. ``--attention-config '{"backend": "FLASH_ATTN"}'``
→ FA4 + rswa_mask_mod. Exact token-level R-SWA.
``flash_attn_version`` is forced to 4 if not already set (R-SWA
mask_mod requires FA4; FA3 cannot express it). Raises if FA4 is
not available on this device.
2. ``--attention-config '{"backend": "FLEX_ATTENTION"}'``
→ FlexAttention R-SWA via Triton block mask.
3. ``--attention-config '{"backend": "auto"}'`` (or omitted)
→ Auto-detect: FA4 if available (H20/H100 SM90), else FlexAttention.
Regardless of backend, prefix caching is disabled for this model: R-SWA
decode-phase KV is not a pure causal function of the prefix (so decode
blocks are not reusable), and single-turn image-led OCR prompts rarely
hit the prefix cache.
Example — force FlexAttention even on a machine with FA4::
vllm serve baidu/Unlimited-OCR \\
--attention-config '{"backend": "FLEX_ATTENTION"}'
"""
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.vllm_flash_attn import is_fa_version_supported
attn_config = vllm_config.attention_config
fa4_available = is_fa_version_supported(4)
# ── step 1: resolve backend ─────────────────────────────────────────
# None means the user did not explicitly specify a backend; auto-select.
if attn_config.backend is None:
attn_config.backend = (
AttentionBackendEnum.FLASH_ATTN
if fa4_available
else AttentionBackendEnum.FLEX_ATTENTION
)
logger.info(
"Unlimited-OCR: auto-selected attention backend=%s (fa4_available=%s).",
attn_config.backend.value,
fa4_available,
)
# ── step 2: configure the chosen backend ────────────────────────────
if attn_config.backend == AttentionBackendEnum.FLASH_ATTN:
if not fa4_available:
raise RuntimeError(
"Unlimited-OCR: --attention-config backend=FLASH_ATTN "
"requires FA4 (rswa_mask_mod), but FA4 is not available on "
"this device/installation. Use backend=FLEX_ATTENTION or "
"upgrade vllm-flash-attn."
)
# On SM90 (H20), the default FA version is FA3 regardless of FA4
# availability (FA4 is only auto-upgraded when head_size > 256).
# The R-SWA mask_mod requires FA4, so force the version globally.
if attn_config.flash_attn_version is None:
attn_config.flash_attn_version = 4
elif attn_config.flash_attn_version < 4:
logger.warning(
"Unlimited-OCR: flash_attn_version=%d cannot express the "
"R-SWA mask_mod; upgrading to 4.",
attn_config.flash_attn_version,
)
attn_config.flash_attn_version = 4
logger.info(
"Unlimited-OCR: FlashAttention FA%d + rswa_mask_mod — exact R-SWA.",
attn_config.flash_attn_version,
)
elif attn_config.backend == AttentionBackendEnum.FLEX_ATTENTION:
logger.info(
"Unlimited-OCR: FlexAttention — R-SWA via Triton block mask%s.",
""
if not fa4_available
else (
" (FA4 available but not used; pass backend=FLASH_ATTN to upgrade)"
),
)
else:
raise ValueError(
f"Unlimited-OCR: unsupported attention backend "
f"{attn_config.backend!r} for R-SWA. "
"Use FLASH_ATTN (FA4) or FLEX_ATTENTION."
)
# R-SWA windows the *generated* tokens, so a decode-token's KV is not a
# pure causal function of the prefix and cannot be safely reused across
# requests via prefix caching. Only the prompt/image prefix is cacheable,
# but OCR is single-turn with image-led prompts that rarely share a
# prefix, so prefix caching brings little benefit while complicating the
# KV cache manager. Disable it for this model.
cache_config = vllm_config.cache_config
if cache_config.enable_prefix_caching:
cache_config.enable_prefix_caching = False
logger.info(
"Unlimited-OCR: disabling prefix caching (R-SWA decode KV is not "
"cacheable, and single-turn image-led prompts rarely hit the "
"prefix cache)."
)
mm_config = getattr(vllm_config.model_config, "multimodal_config", None)
if mm_config is not None:
if mm_config.mm_encoder_attn_backend is None:
mm_config.mm_encoder_attn_backend = AttentionBackendEnum.FLASH_ATTN
elif mm_config.mm_encoder_attn_backend == AttentionBackendEnum.FLASHINFER:
logger.warning(
"Unlimited-OCR: FlashInfer is not supported for the vision "
"encoder (the CLIP stage runs full attention without "
"cu_seqlens); falling back to FlashAttention."
)
mm_config.mm_encoder_attn_backend = AttentionBackendEnum.FLASH_ATTN
@staticmethod
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
text_config = model_config.hf_config.text_config
text_config.architectures = ["DeepseekV2ForCausalLM"]
if getattr(model_config.hf_config, "rswa_window", None) is None:
model_config.hf_config.rswa_window = 128
# Propagate rswa_window to text_config so that DeepseekAttention (which
# receives text_config as its vllm_config.model_config.hf_config via
# init_vllm_registered_model) can read it and create RSWAAttention.
rswa_window = model_config.hf_config.rswa_window
text_config.rswa_window = rswa_window
class Gemma4Config(VerifyAndUpdateConfig):
@staticmethod
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
@@ -703,6 +836,7 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
"Qwen3VLForSequenceClassification": Qwen3VLForSequenceClassificationConfig,
"Qwen3_5ForConditionalGeneration": Qwen3_5ForConditionalGenerationConfig,
"Qwen3_5MoeForConditionalGeneration": Qwen3_5ForConditionalGenerationConfig,
"UnlimitedOCRForCausalLM": UnlimitedOCRForCausalLMConfig,
"VoyageQwen3BidirectionalEmbedModel": VoyageQwen3BidirectionalEmbedModelConfig,
"XLMRobertaModel": JinaRobertaModelConfig,
}
+4 -2
View File
@@ -220,8 +220,10 @@ class DeepseekOCRProcessingInfo(BaseProcessingInfo):
patch_size = 16
downsample_ratio = 4
if CROP_MODE:
if image_width <= 640 and image_height <= 640:
# Use the caller-supplied `cropping` flag so that callers that disable
# crop mode for multi-image requests get a consistent token count.
if cropping:
if image_width <= IMAGE_SIZE and image_height <= IMAGE_SIZE:
crop_ratio = [1, 1]
else:
# find the closest aspect ratio to the target
+25 -12
View File
@@ -45,7 +45,7 @@ from vllm.distributed import (
)
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.attention import Attention, RSWAAttention
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
@@ -174,15 +174,28 @@ class DeepseekAttention(nn.Module):
max_position=max_position_embeddings,
rope_parameters=config.rope_parameters,
)
self.attn = Attention(
self.num_heads,
self.head_dim,
self.scaling,
num_kv_heads=self.num_kv_heads,
cache_config=cache_config,
quant_config=quant_config,
prefix=f"{prefix}.attn",
)
rswa_window = getattr(vllm_config.model_config.hf_config, "rswa_window", None)
if rswa_window is not None:
self.attn = RSWAAttention(
self.num_heads,
self.head_dim,
self.scaling,
num_kv_heads=self.num_kv_heads,
cache_config=cache_config,
quant_config=quant_config,
prefix=f"{prefix}.attn",
rswa_window=rswa_window,
)
else:
self.attn = Attention(
self.num_heads,
self.head_dim,
self.scaling,
num_kv_heads=self.num_kv_heads,
cache_config=cache_config,
quant_config=quant_config,
prefix=f"{prefix}.attn",
)
def forward(
self,
@@ -588,12 +601,12 @@ class DeepseekV32IndexerCache(torch.nn.Module, AttentionLayerBase):
compilation_config.static_forward_context[prefix] = self
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
return MLAAttentionSpec( # Only has one vector instead of K + V
return MLAAttentionSpec(
block_size=self.cache_config.block_size,
num_kv_heads=1,
head_size=self.head_dim,
dtype=self.dtype,
)
) # Only has one vector instead of K + V
def forward(self): ...
+1
View File
@@ -358,6 +358,7 @@ _MULTIMODAL_MODELS = {
"DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
"DeepseekOCRForCausalLM": ("deepseek_ocr", "DeepseekOCRForCausalLM"),
"DeepseekOCR2ForCausalLM": ("deepseek_ocr2", "DeepseekOCR2ForCausalLM"),
"UnlimitedOCRForCausalLM": ("unlimited_ocr", "UnlimitedOCRForCausalLM"),
"DotsOCRForCausalLM": ("dots_ocr", "DotsOCRForCausalLM"),
"Eagle2_5_VLForConditionalGeneration": (
"eagle2_5_vl",
+250
View File
@@ -0,0 +1,250 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inference-only Unlimited-OCR model compatible with HuggingFace weights.
Unlimited-OCR (``baidu/Unlimited-OCR``) shares
the exact DeepSeek-OCR (gundam, ``base_size=1024`` / ``image_size=640`` / crop)
vision stack: a DeepEncoder (SAM-ViT-B + CLIP-L) followed by a linear MLP
projector, with the same image-token tiling layout. The only difference is the
language backbone, which is a DeepSeek-V2 *MoE* (64 routed + 2 shared experts,
``first_k_dense_replace=1``) that uses plain multi-head attention
(``use_mla=False``, ``qk_nope_head_dim == qk_rope_head_dim == 0``) instead of
the dense MLA decoder used by DeepSeek-OCR.
vLLM's ``DeepseekV2DecoderLayer`` already dispatches to the plain-MHA
``DeepseekAttention`` whenever ``qk_nope_head_dim == qk_rope_head_dim == 0`` and
builds the MoE blocks straight from the config, so the whole DeepSeek-OCR
multimodal wrapper can be reused verbatim. Model-specific config (language
backbone architecture, FlexAttention for R-SWA, vision encoder backend, and
``rswa_window``) is applied in ``UnlimitedOCRForCausalLMConfig``.
Attention backend: the reference applies Reference Sliding Window Attention
(R-SWA) -- the prompt/image tokens form a globally-visible prefix while the
*generated* tokens additionally attend only a fixed sliding window (128) of
recent tokens. We reproduce this (Level 1: full KV cache + custom mask) by
forcing the language model onto the FlexAttention backend and installing an
R-SWA ``mask_mod``. FlexAttention is the only backend able to express the
"global prefix + sliding window" mask; FlashAttention-3 / Triton only support a
uniform window (and additionally crash or compute incorrectly on this decoder's
10-head,
head_dim-128 shape), and FlashInfer's paged decode exposes no custom mask. The
window size is published via ``model_config.rswa_window``, which the model
runner reads to plumb per-request prefix lengths into the FlexAttention mask.
The *vision encoder* (DeepEncoder's CLIP stage, head_dim 64) is unaffected and
does not use R-SWA: it runs a single full-attention prefill pass. FlashAttention,
Triton and torch SDPA all produce correct, equally fast results; only FlashInfer
is incompatible (its ViT path asserts on the varlen cu_seqlens metadata that this
CLIP encoder never builds). We default the encoder to FlashAttention and
transparently fall back to it if FlashInfer is requested.
To suppress repetition on long documents, use ``NGramPerReqLogitsProcessor`` from
this module (same request-level processor as DeepSeek-OCR) with::
SamplingParams(
temperature=0.0,
max_tokens=8192,
extra_args={"ngram_size": 35, "window_size": 128},
)
Image processing
----------------
Unlimited-OCR supports up to 32 local crops (vs 6 for DeepSeek-OCR), i.e.
``dynamic_preprocess`` runs with ``max_num=32``.
Multi-image requests fall back to non-crop mode: crop ("gundam") mode is only
used for single-image input. DeepSeek-OCR does *not* have this restriction.
Because that fallback makes the per-image processor output depend on *how many*
images are in the request, it breaks the assumption behind vLLM's per-item
multimodal processing cache (``MultiModalProcessorOnlyCache``). We handle this
the same way ``DeepseekVL2MultiModalProcessor`` does: only the single-image case
(which always crops) is cached, while multi-image requests bypass the cache and
are recomputed fresh -- see ``_cached_apply_hf_processor`` below. This keeps the
processing cache consistent (verified by ``test_processing_correctness``).
"""
import math
from collections.abc import Mapping, Sequence
from vllm.config import VllmConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalKwargsItems
from vllm.multimodal.parse import (
ImageEmbeddingItems,
ImageProcessorItems,
ImageSize,
MultiModalDataItems,
)
from vllm.multimodal.processing import PromptReplacement, PromptUpdate
from vllm.multimodal.processing.context import TimingContext
from vllm.multimodal.processing.inputs import ProcessorInputs
from vllm.multimodal.processing.processor import MultiModalProcessingInfo
from vllm.transformers_utils.processors.deepseek_ocr import (
BASE_SIZE,
CROP_MODE,
IMAGE_SIZE,
count_tiles,
)
from .deepseek_ocr import (
DeepseekOCRDummyInputsBuilder,
DeepseekOCRForCausalLM,
DeepseekOCRMultiModalProcessor,
DeepseekOCRProcessingInfo,
NGramPerReqLogitsProcessor,
)
__all__ = [
"NGramPerReqLogitsProcessor",
"UnlimitedOCRForCausalLM",
]
# Unlimited-OCR supports up to 32 local crops (vs 6 for DeepSeek-OCR).
_UNLIMITED_OCR_MAX_CROPS = 32
class UnlimitedOCRProcessingInfo(DeepseekOCRProcessingInfo):
"""ProcessingInfo for Unlimited-OCR: same as DeepSeek-OCR but with
max_crops=32 instead of 6. The higher crop count allows tiling very large
document pages into up to 32 640×640 patches (dynamic_preprocess max_num=32).
"""
def get_hf_config(self):
from vllm.transformers_utils.configs.unlimited_ocr import UnlimitedOCRConfig
return self.ctx.get_hf_config(UnlimitedOCRConfig)
def get_hf_processor(self, **kwargs: object):
from vllm.transformers_utils.processors.unlimited_ocr import (
UnlimitedOCRProcessor,
)
v1_processor_config = dict(
image_size=IMAGE_SIZE,
base_size=BASE_SIZE,
crop_mode=CROP_MODE,
strategy="v1",
max_crops=_UNLIMITED_OCR_MAX_CROPS,
)
return self.ctx.get_hf_processor(
UnlimitedOCRProcessor,
**{**v1_processor_config, **kwargs},
)
def get_num_image_tokens(
self, *, image_width: int, image_height: int, cropping: bool = True
) -> int:
patch_size = 16
downsample_ratio = 4
# Honour the caller-supplied `cropping` flag: multi-image callers pass
# cropping=False to match UnlimitedOCRProcessor.tokenize_with_images.
if cropping:
if image_width <= IMAGE_SIZE and image_height <= IMAGE_SIZE:
crop_ratio = [1, 1]
else:
crop_ratio = count_tiles(
image_width,
image_height,
max_num=_UNLIMITED_OCR_MAX_CROPS,
image_size=IMAGE_SIZE,
)
num_width_tiles, num_height_tiles = crop_ratio
else:
num_width_tiles = num_height_tiles = 1
h = w = math.ceil((BASE_SIZE // patch_size) / downsample_ratio)
h2 = w2 = math.ceil((IMAGE_SIZE // patch_size) / downsample_ratio)
global_views_tokens = h * (w + 1)
if num_width_tiles > 1 or num_height_tiles > 1:
local_views_tokens = (num_height_tiles * h2) * (num_width_tiles * w2 + 1)
else:
local_views_tokens = 0
return global_views_tokens + local_views_tokens + 1
def get_image_size_with_most_features(self) -> ImageSize:
# With max_crops=32, the widest possible grid is 4×8 (aspect ratio 1:2).
# A 2560×5120 image (4×640 × 8×640) selects exactly 4×8=32 tiles and
# produces the maximum token count.
return ImageSize(width=640 * 4, height=640 * 8)
class UnlimitedOCRMultiModalProcessor(DeepseekOCRMultiModalProcessor):
"""Multimodal processor for Unlimited-OCR.
Disables crop mode for multi-image requests (to stay consistent with
``UnlimitedOCRProcessor.tokenize_with_images``), and -- since that makes the
per-image output depend on the request's image count -- bypasses the
per-item processing cache for multi-image requests, exactly like
``DeepseekVL2MultiModalProcessor``.
DeepSeek-OCR does *not* apply either of these.
"""
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
image_token_id = hf_processor.image_token_id
assert isinstance(image_token_id, int)
def get_replacement_unlimited_ocr(item_idx: int):
images = mm_items.get_items(
"image", (ImageEmbeddingItems, ImageProcessorItems)
)
if isinstance(images, ImageEmbeddingItems):
num_image_tokens = images.get_feature_size(item_idx)
else:
size = images.get_image_size(item_idx)
# Disable crop mode for multi-image input.
# UnlimitedOCRProcessor.tokenize_with_images applies the same
# fallback, so both paths must agree on the effective crop flag.
effective_cropping = CROP_MODE and len(images) == 1
num_image_tokens = self.info.get_num_image_tokens(
image_width=size.width,
image_height=size.height,
cropping=effective_cropping,
)
return [image_token_id] * num_image_tokens
return [
PromptReplacement(
modality="image",
target=[image_token_id],
replacement=get_replacement_unlimited_ocr,
)
]
def _cached_apply_hf_processor(
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
# The processor logic differs for single-image (crop) vs multi-image
# (no crop) requests. The processing cache assumes per-item output is
# invariant of how many images are passed per prompt, so we only cache
# the single-image case and recompute multi-image requests fresh.
if inputs.mm_data_items.get_count("image", strict=False) > 1:
return self._apply_hf_processor(inputs, timing_ctx)
return super()._cached_apply_hf_processor(inputs, timing_ctx)
@MULTIMODAL_REGISTRY.register_processor(
UnlimitedOCRMultiModalProcessor,
info=UnlimitedOCRProcessingInfo,
dummy_inputs=DeepseekOCRDummyInputsBuilder,
)
class UnlimitedOCRForCausalLM(DeepseekOCRForCausalLM):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__(vllm_config=vllm_config, prefix=prefix)