[Model] support Qianfan-OCR model (#40136)

Signed-off-by: bairongz <baiyuu.cs@gmail.com>
Signed-off-by: zhuangbairong <zhuangbairong@baidu.com>
Co-authored-by: zhuangbairong <zhuangbairong@baidu.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
bairongz
2026-05-05 10:51:25 +00:00
committed by GitHub
co-authored by zhuangbairong gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent 8b9ea2f881
commit 0a201b60cf
9 changed files with 312 additions and 0 deletions
+92
View File
@@ -0,0 +1,92 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# QianfanOCR is built on InternVL with a Qwen3 language backbone.
# The model architecture and weights are fully compatible with InternVLChatModel,
# only the config model_type / architectures strings differ.
from transformers import PretrainedConfig
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.transformers_utils.processors.internvl import (
InternVLImageProcessor,
InternVLProcessor,
)
from .internvl import (
BaseInternVLDummyInputsBuilder,
BaseInternVLMultiModalProcessor,
BaseInternVLProcessingInfo,
InternVLChatModel,
)
class QianfanOCRProcessingInfo(BaseInternVLProcessingInfo):
"""Image-only ProcessingInfo for QianfanOCR (no video support)."""
def get_hf_processor(self, **kwargs: object) -> InternVLProcessor:
config = self.get_hf_config()
vision_config = config.vision_config
kwargs = self.ctx.get_merged_mm_kwargs(kwargs)
kwargs.setdefault("image_size", vision_config.image_size)
kwargs.setdefault("min_dynamic_patch", config.min_dynamic_patch)
kwargs.setdefault("max_dynamic_patch", config.max_dynamic_patch)
kwargs.setdefault("dynamic_image_size", config.dynamic_image_size)
kwargs.setdefault("use_thumbnail", config.use_thumbnail)
image_processor = InternVLImageProcessor(**kwargs)
image_size = image_processor.image_size
patch_size = vision_config.patch_size
downsample_ratio = config.downsample_ratio
image_seq_length = int((image_size // patch_size) ** 2 * (downsample_ratio**2))
return InternVLProcessor(
tokenizer=self.get_tokenizer(),
image_processor=image_processor,
video_processor=None,
image_seq_length=image_seq_length,
ctx_video_token=None,
)
@MULTIMODAL_REGISTRY.register_processor(
BaseInternVLMultiModalProcessor,
info=QianfanOCRProcessingInfo,
dummy_inputs=BaseInternVLDummyInputsBuilder,
)
class QianfanOCRForConditionalGeneration(InternVLChatModel):
"""QianfanOCR multimodal model.
Identical in structure to InternVLChatModel (InternViT vision encoder +
pixel-shuffle MLP connector + Qwen3 language model). This class exists
solely to register the ``QianfanOCRForConditionalGeneration`` architecture
name that appears in the model's config.json.
"""
def _patch_quant_config(
self, config: PretrainedConfig, quant_config: QuantizationConfig
) -> None:
super()._patch_quant_config(config, quant_config)
# ignore vit layers to preserve model performance
if isinstance(quant_config, Fp8Config):
_FP8_IGNORED_LAYERS = [
*(
layer
for i in range(config.vision_config.num_hidden_layers)
for layer in [
f"vision_model.encoder.layers.{i}.attn.qkv",
f"vision_model.encoder.layers.{i}.attn.proj",
f"vision_model.encoder.layers.{i}.mlp.fc1",
f"vision_model.encoder.layers.{i}.mlp.fc2",
]
),
"language_model.lm_head",
"mlp1.1",
"mlp1.3",
]
for layer in _FP8_IGNORED_LAYERS:
if layer not in quant_config.ignored_layers:
quant_config.ignored_layers.append(layer)
+4
View File
@@ -511,6 +511,10 @@ _MULTIMODAL_MODELS = {
"Phi4ForCausalLMV": ("phi4siglip", "Phi4ForCausalLMV"),
"Phi4MMForCausalLM": ("phi4mm", "Phi4MMForCausalLM"),
"PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"),
"QianfanOCRForConditionalGeneration": (
"qianfan_ocr",
"QianfanOCRForConditionalGeneration",
),
"QwenVLForConditionalGeneration": ("qwen_vl", "QwenVLForConditionalGeneration"),
"Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),
"Qwen2_5_VLForConditionalGeneration": (
+1
View File
@@ -125,6 +125,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
step3_vl="Step3VLConfig",
step3_text="Step3TextConfig",
step3p5="Step3p5Config",
qianfan_ocr="QianfanOCRConfig",
qwen3_asr="Qwen3ASRConfig",
qwen3_next="Qwen3NextConfig",
qwen3_5="Qwen3_5Config",
@@ -70,6 +70,8 @@ _CLASS_TO_MODULE: dict[str, str] = {
"Step3VisionEncoderConfig": "vllm.transformers_utils.configs.step3_vl",
"Step3TextConfig": "vllm.transformers_utils.configs.step3_vl",
"Step3p5Config": "vllm.transformers_utils.configs.step3p5",
"QianfanOCRConfig": "vllm.transformers_utils.configs.qianfan_ocr",
"QianfanOCRVisionConfig": "vllm.transformers_utils.configs.qianfan_ocr",
"Qwen3ASRConfig": "vllm.transformers_utils.configs.qwen3_asr",
"Qwen3NextConfig": "vllm.transformers_utils.configs.qwen3_next",
"Qwen3_5Config": "vllm.transformers_utils.configs.qwen3_5",
@@ -135,6 +137,8 @@ __all__ = [
"Step3VisionEncoderConfig",
"Step3TextConfig",
"Step3p5Config",
"QianfanOCRConfig",
"QianfanOCRVisionConfig",
"Qwen3ASRConfig",
"Qwen3NextConfig",
"Qwen3_5Config",
@@ -0,0 +1,105 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
from transformers import PretrainedConfig
from transformers.models.auto import CONFIG_MAPPING
class QianfanOCRVisionConfig(PretrainedConfig):
model_type = "qianfan_ocr_vision"
def __init__(
self,
hidden_size: int = 1024,
intermediate_size: int = 4096,
num_hidden_layers: int = 24,
num_attention_heads: int = 16,
num_channels: int = 3,
image_size: int = 448,
patch_size: int = 14,
hidden_act: str = "gelu",
layer_norm_eps: float = 1e-6,
attention_dropout: float = 0.0,
drop_path_rate: float = 0.1,
qkv_bias: bool = True,
qk_normalization: bool = False,
norm_type: str = "layer_norm",
initializer_range: float = 0.02,
initializer_factor: float = 0.1,
use_mask_token: bool = False,
use_mean_pooling: bool = True,
**kwargs: Any,
):
super().__init__(**kwargs)
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.num_hidden_layers = num_hidden_layers
self.num_attention_heads = num_attention_heads
self.num_channels = num_channels
self.image_size = image_size
self.patch_size = patch_size
self.hidden_act = hidden_act
self.layer_norm_eps = layer_norm_eps
self.attention_dropout = attention_dropout
self.drop_path_rate = drop_path_rate
self.qkv_bias = qkv_bias
self.qk_normalization = qk_normalization
self.norm_type = norm_type
self.initializer_range = initializer_range
self.initializer_factor = initializer_factor
self.use_mask_token = use_mask_token
self.use_mean_pooling = use_mean_pooling
class QianfanOCRConfig(PretrainedConfig):
model_type = "qianfan_ocr"
def __init__(
self,
vision_config: dict | None = None,
text_config: dict | None = None,
downsample_ratio: float = 0.5,
dynamic_image_size: bool = True,
force_image_size: int = 448,
image_token_id: int = 151671,
max_dynamic_patch: int = 12,
min_dynamic_patch: int = 1,
pad2square: bool = False,
ps_version: str = "v2",
select_layer: int = -1,
template: str = "internvl2_5",
use_thumbnail: bool = True,
tie_word_embeddings: bool = False,
**kwargs: Any,
):
super().__init__(**kwargs)
if isinstance(vision_config, dict):
self.vision_config = QianfanOCRVisionConfig(**vision_config)
elif vision_config is None:
self.vision_config = QianfanOCRVisionConfig()
else:
self.vision_config = vision_config
if isinstance(text_config, dict):
model_type = text_config.get("model_type", "qwen3")
self.text_config = CONFIG_MAPPING[model_type](**text_config)
elif text_config is None:
self.text_config = CONFIG_MAPPING["qwen3"]()
else:
self.text_config = text_config
self.downsample_ratio = downsample_ratio
self.dynamic_image_size = dynamic_image_size
self.force_image_size = force_image_size
self.image_token_id = image_token_id
self.max_dynamic_patch = max_dynamic_patch
self.min_dynamic_patch = min_dynamic_patch
self.pad2square = pad2square
self.ps_version = ps_version
self.select_layer = select_layer
self.template = template
self.use_thumbnail = use_thumbnail
self.tie_word_embeddings = tie_word_embeddings