[Model] Support Unlimited OCR (#46564)

Signed-off-by: Tianyu Guo <guoty9@mail2.sysu.edu.cn>
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
Tianyu Guo
2026-06-27 23:09:18 -07:00
committed by GitHub
co-authored by Isotr0py Roger Wang
parent a65f93fb2e
commit c6741b2ad4
35 changed files with 1084 additions and 49 deletions
+134
View File
@@ -54,6 +54,139 @@ class Gemma3TextModelConfig(VerifyAndUpdateConfig):
hf_config.is_causal = not hf_config.use_bidirectional_attention
class UnlimitedOCRForCausalLMConfig(VerifyAndUpdateConfig):
@staticmethod
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
"""Configure Unlimited-OCR attention backends for R-SWA and vision.
Backend selection — controlled by the standard ``--attention-config``
CLI argument (priority order):
1. ``--attention-config '{"backend": "FLASH_ATTN"}'``
→ FA4 + rswa_mask_mod. Exact token-level R-SWA.
``flash_attn_version`` is forced to 4 if not already set (R-SWA
mask_mod requires FA4; FA3 cannot express it). Raises if FA4 is
not available on this device.
2. ``--attention-config '{"backend": "FLEX_ATTENTION"}'``
→ FlexAttention R-SWA via Triton block mask.
3. ``--attention-config '{"backend": "auto"}'`` (or omitted)
→ Auto-detect: FA4 if available (H20/H100 SM90), else FlexAttention.
Regardless of backend, prefix caching is disabled for this model: R-SWA
decode-phase KV is not a pure causal function of the prefix (so decode
blocks are not reusable), and single-turn image-led OCR prompts rarely
hit the prefix cache.
Example — force FlexAttention even on a machine with FA4::
vllm serve baidu/Unlimited-OCR \\
--attention-config '{"backend": "FLEX_ATTENTION"}'
"""
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.vllm_flash_attn import is_fa_version_supported
attn_config = vllm_config.attention_config
fa4_available = is_fa_version_supported(4)
# ── step 1: resolve backend ─────────────────────────────────────────
# None means the user did not explicitly specify a backend; auto-select.
if attn_config.backend is None:
attn_config.backend = (
AttentionBackendEnum.FLASH_ATTN
if fa4_available
else AttentionBackendEnum.FLEX_ATTENTION
)
logger.info(
"Unlimited-OCR: auto-selected attention backend=%s (fa4_available=%s).",
attn_config.backend.value,
fa4_available,
)
# ── step 2: configure the chosen backend ────────────────────────────
if attn_config.backend == AttentionBackendEnum.FLASH_ATTN:
if not fa4_available:
raise RuntimeError(
"Unlimited-OCR: --attention-config backend=FLASH_ATTN "
"requires FA4 (rswa_mask_mod), but FA4 is not available on "
"this device/installation. Use backend=FLEX_ATTENTION or "
"upgrade vllm-flash-attn."
)
# On SM90 (H20), the default FA version is FA3 regardless of FA4
# availability (FA4 is only auto-upgraded when head_size > 256).
# The R-SWA mask_mod requires FA4, so force the version globally.
if attn_config.flash_attn_version is None:
attn_config.flash_attn_version = 4
elif attn_config.flash_attn_version < 4:
logger.warning(
"Unlimited-OCR: flash_attn_version=%d cannot express the "
"R-SWA mask_mod; upgrading to 4.",
attn_config.flash_attn_version,
)
attn_config.flash_attn_version = 4
logger.info(
"Unlimited-OCR: FlashAttention FA%d + rswa_mask_mod — exact R-SWA.",
attn_config.flash_attn_version,
)
elif attn_config.backend == AttentionBackendEnum.FLEX_ATTENTION:
logger.info(
"Unlimited-OCR: FlexAttention — R-SWA via Triton block mask%s.",
""
if not fa4_available
else (
" (FA4 available but not used; pass backend=FLASH_ATTN to upgrade)"
),
)
else:
raise ValueError(
f"Unlimited-OCR: unsupported attention backend "
f"{attn_config.backend!r} for R-SWA. "
"Use FLASH_ATTN (FA4) or FLEX_ATTENTION."
)
# R-SWA windows the *generated* tokens, so a decode-token's KV is not a
# pure causal function of the prefix and cannot be safely reused across
# requests via prefix caching. Only the prompt/image prefix is cacheable,
# but OCR is single-turn with image-led prompts that rarely share a
# prefix, so prefix caching brings little benefit while complicating the
# KV cache manager. Disable it for this model.
cache_config = vllm_config.cache_config
if cache_config.enable_prefix_caching:
cache_config.enable_prefix_caching = False
logger.info(
"Unlimited-OCR: disabling prefix caching (R-SWA decode KV is not "
"cacheable, and single-turn image-led prompts rarely hit the "
"prefix cache)."
)
mm_config = getattr(vllm_config.model_config, "multimodal_config", None)
if mm_config is not None:
if mm_config.mm_encoder_attn_backend is None:
mm_config.mm_encoder_attn_backend = AttentionBackendEnum.FLASH_ATTN
elif mm_config.mm_encoder_attn_backend == AttentionBackendEnum.FLASHINFER:
logger.warning(
"Unlimited-OCR: FlashInfer is not supported for the vision "
"encoder (the CLIP stage runs full attention without "
"cu_seqlens); falling back to FlashAttention."
)
mm_config.mm_encoder_attn_backend = AttentionBackendEnum.FLASH_ATTN
@staticmethod
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
text_config = model_config.hf_config.text_config
text_config.architectures = ["DeepseekV2ForCausalLM"]
if getattr(model_config.hf_config, "rswa_window", None) is None:
model_config.hf_config.rswa_window = 128
# Propagate rswa_window to text_config so that DeepseekAttention (which
# receives text_config as its vllm_config.model_config.hf_config via
# init_vllm_registered_model) can read it and create RSWAAttention.
rswa_window = model_config.hf_config.rswa_window
text_config.rswa_window = rswa_window
class Gemma4Config(VerifyAndUpdateConfig):
@staticmethod
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
@@ -703,6 +836,7 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
"Qwen3VLForSequenceClassification": Qwen3VLForSequenceClassificationConfig,
"Qwen3_5ForConditionalGeneration": Qwen3_5ForConditionalGenerationConfig,
"Qwen3_5MoeForConditionalGeneration": Qwen3_5ForConditionalGenerationConfig,
"UnlimitedOCRForCausalLM": UnlimitedOCRForCausalLMConfig,
"VoyageQwen3BidirectionalEmbedModel": VoyageQwen3BidirectionalEmbedModelConfig,
"XLMRobertaModel": JinaRobertaModelConfig,
}