[Model] support Qianfan-OCR model (#40136)
Signed-off-by: bairongz <baiyuu.cs@gmail.com> Signed-off-by: zhuangbairong <zhuangbairong@baidu.com> Co-authored-by: zhuangbairong <zhuangbairong@baidu.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
zhuangbairong
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent
8b9ea2f881
commit
0a201b60cf
@@ -0,0 +1,92 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# QianfanOCR is built on InternVL with a Qwen3 language backbone.
|
||||
# The model architecture and weights are fully compatible with InternVLChatModel,
|
||||
# only the config model_type / architectures strings differ.
|
||||
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.transformers_utils.processors.internvl import (
|
||||
InternVLImageProcessor,
|
||||
InternVLProcessor,
|
||||
)
|
||||
|
||||
from .internvl import (
|
||||
BaseInternVLDummyInputsBuilder,
|
||||
BaseInternVLMultiModalProcessor,
|
||||
BaseInternVLProcessingInfo,
|
||||
InternVLChatModel,
|
||||
)
|
||||
|
||||
|
||||
class QianfanOCRProcessingInfo(BaseInternVLProcessingInfo):
|
||||
"""Image-only ProcessingInfo for QianfanOCR (no video support)."""
|
||||
|
||||
def get_hf_processor(self, **kwargs: object) -> InternVLProcessor:
|
||||
config = self.get_hf_config()
|
||||
vision_config = config.vision_config
|
||||
|
||||
kwargs = self.ctx.get_merged_mm_kwargs(kwargs)
|
||||
kwargs.setdefault("image_size", vision_config.image_size)
|
||||
kwargs.setdefault("min_dynamic_patch", config.min_dynamic_patch)
|
||||
kwargs.setdefault("max_dynamic_patch", config.max_dynamic_patch)
|
||||
kwargs.setdefault("dynamic_image_size", config.dynamic_image_size)
|
||||
kwargs.setdefault("use_thumbnail", config.use_thumbnail)
|
||||
|
||||
image_processor = InternVLImageProcessor(**kwargs)
|
||||
image_size = image_processor.image_size
|
||||
patch_size = vision_config.patch_size
|
||||
downsample_ratio = config.downsample_ratio
|
||||
image_seq_length = int((image_size // patch_size) ** 2 * (downsample_ratio**2))
|
||||
|
||||
return InternVLProcessor(
|
||||
tokenizer=self.get_tokenizer(),
|
||||
image_processor=image_processor,
|
||||
video_processor=None,
|
||||
image_seq_length=image_seq_length,
|
||||
ctx_video_token=None,
|
||||
)
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
BaseInternVLMultiModalProcessor,
|
||||
info=QianfanOCRProcessingInfo,
|
||||
dummy_inputs=BaseInternVLDummyInputsBuilder,
|
||||
)
|
||||
class QianfanOCRForConditionalGeneration(InternVLChatModel):
|
||||
"""QianfanOCR multimodal model.
|
||||
|
||||
Identical in structure to InternVLChatModel (InternViT vision encoder +
|
||||
pixel-shuffle MLP connector + Qwen3 language model). This class exists
|
||||
solely to register the ``QianfanOCRForConditionalGeneration`` architecture
|
||||
name that appears in the model's config.json.
|
||||
"""
|
||||
|
||||
def _patch_quant_config(
|
||||
self, config: PretrainedConfig, quant_config: QuantizationConfig
|
||||
) -> None:
|
||||
super()._patch_quant_config(config, quant_config)
|
||||
# ignore vit layers to preserve model performance
|
||||
if isinstance(quant_config, Fp8Config):
|
||||
_FP8_IGNORED_LAYERS = [
|
||||
*(
|
||||
layer
|
||||
for i in range(config.vision_config.num_hidden_layers)
|
||||
for layer in [
|
||||
f"vision_model.encoder.layers.{i}.attn.qkv",
|
||||
f"vision_model.encoder.layers.{i}.attn.proj",
|
||||
f"vision_model.encoder.layers.{i}.mlp.fc1",
|
||||
f"vision_model.encoder.layers.{i}.mlp.fc2",
|
||||
]
|
||||
),
|
||||
"language_model.lm_head",
|
||||
"mlp1.1",
|
||||
"mlp1.3",
|
||||
]
|
||||
for layer in _FP8_IGNORED_LAYERS:
|
||||
if layer not in quant_config.ignored_layers:
|
||||
quant_config.ignored_layers.append(layer)
|
||||
@@ -511,6 +511,10 @@ _MULTIMODAL_MODELS = {
|
||||
"Phi4ForCausalLMV": ("phi4siglip", "Phi4ForCausalLMV"),
|
||||
"Phi4MMForCausalLM": ("phi4mm", "Phi4MMForCausalLM"),
|
||||
"PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"),
|
||||
"QianfanOCRForConditionalGeneration": (
|
||||
"qianfan_ocr",
|
||||
"QianfanOCRForConditionalGeneration",
|
||||
),
|
||||
"QwenVLForConditionalGeneration": ("qwen_vl", "QwenVLForConditionalGeneration"),
|
||||
"Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),
|
||||
"Qwen2_5_VLForConditionalGeneration": (
|
||||
|
||||
@@ -125,6 +125,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
|
||||
step3_vl="Step3VLConfig",
|
||||
step3_text="Step3TextConfig",
|
||||
step3p5="Step3p5Config",
|
||||
qianfan_ocr="QianfanOCRConfig",
|
||||
qwen3_asr="Qwen3ASRConfig",
|
||||
qwen3_next="Qwen3NextConfig",
|
||||
qwen3_5="Qwen3_5Config",
|
||||
|
||||
@@ -70,6 +70,8 @@ _CLASS_TO_MODULE: dict[str, str] = {
|
||||
"Step3VisionEncoderConfig": "vllm.transformers_utils.configs.step3_vl",
|
||||
"Step3TextConfig": "vllm.transformers_utils.configs.step3_vl",
|
||||
"Step3p5Config": "vllm.transformers_utils.configs.step3p5",
|
||||
"QianfanOCRConfig": "vllm.transformers_utils.configs.qianfan_ocr",
|
||||
"QianfanOCRVisionConfig": "vllm.transformers_utils.configs.qianfan_ocr",
|
||||
"Qwen3ASRConfig": "vllm.transformers_utils.configs.qwen3_asr",
|
||||
"Qwen3NextConfig": "vllm.transformers_utils.configs.qwen3_next",
|
||||
"Qwen3_5Config": "vllm.transformers_utils.configs.qwen3_5",
|
||||
@@ -135,6 +137,8 @@ __all__ = [
|
||||
"Step3VisionEncoderConfig",
|
||||
"Step3TextConfig",
|
||||
"Step3p5Config",
|
||||
"QianfanOCRConfig",
|
||||
"QianfanOCRVisionConfig",
|
||||
"Qwen3ASRConfig",
|
||||
"Qwen3NextConfig",
|
||||
"Qwen3_5Config",
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Any
|
||||
|
||||
from transformers import PretrainedConfig
|
||||
from transformers.models.auto import CONFIG_MAPPING
|
||||
|
||||
|
||||
class QianfanOCRVisionConfig(PretrainedConfig):
|
||||
model_type = "qianfan_ocr_vision"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int = 1024,
|
||||
intermediate_size: int = 4096,
|
||||
num_hidden_layers: int = 24,
|
||||
num_attention_heads: int = 16,
|
||||
num_channels: int = 3,
|
||||
image_size: int = 448,
|
||||
patch_size: int = 14,
|
||||
hidden_act: str = "gelu",
|
||||
layer_norm_eps: float = 1e-6,
|
||||
attention_dropout: float = 0.0,
|
||||
drop_path_rate: float = 0.1,
|
||||
qkv_bias: bool = True,
|
||||
qk_normalization: bool = False,
|
||||
norm_type: str = "layer_norm",
|
||||
initializer_range: float = 0.02,
|
||||
initializer_factor: float = 0.1,
|
||||
use_mask_token: bool = False,
|
||||
use_mean_pooling: bool = True,
|
||||
**kwargs: Any,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.num_channels = num_channels
|
||||
self.image_size = image_size
|
||||
self.patch_size = patch_size
|
||||
self.hidden_act = hidden_act
|
||||
self.layer_norm_eps = layer_norm_eps
|
||||
self.attention_dropout = attention_dropout
|
||||
self.drop_path_rate = drop_path_rate
|
||||
self.qkv_bias = qkv_bias
|
||||
self.qk_normalization = qk_normalization
|
||||
self.norm_type = norm_type
|
||||
self.initializer_range = initializer_range
|
||||
self.initializer_factor = initializer_factor
|
||||
self.use_mask_token = use_mask_token
|
||||
self.use_mean_pooling = use_mean_pooling
|
||||
|
||||
|
||||
class QianfanOCRConfig(PretrainedConfig):
|
||||
model_type = "qianfan_ocr"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vision_config: dict | None = None,
|
||||
text_config: dict | None = None,
|
||||
downsample_ratio: float = 0.5,
|
||||
dynamic_image_size: bool = True,
|
||||
force_image_size: int = 448,
|
||||
image_token_id: int = 151671,
|
||||
max_dynamic_patch: int = 12,
|
||||
min_dynamic_patch: int = 1,
|
||||
pad2square: bool = False,
|
||||
ps_version: str = "v2",
|
||||
select_layer: int = -1,
|
||||
template: str = "internvl2_5",
|
||||
use_thumbnail: bool = True,
|
||||
tie_word_embeddings: bool = False,
|
||||
**kwargs: Any,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
|
||||
if isinstance(vision_config, dict):
|
||||
self.vision_config = QianfanOCRVisionConfig(**vision_config)
|
||||
elif vision_config is None:
|
||||
self.vision_config = QianfanOCRVisionConfig()
|
||||
else:
|
||||
self.vision_config = vision_config
|
||||
|
||||
if isinstance(text_config, dict):
|
||||
model_type = text_config.get("model_type", "qwen3")
|
||||
self.text_config = CONFIG_MAPPING[model_type](**text_config)
|
||||
elif text_config is None:
|
||||
self.text_config = CONFIG_MAPPING["qwen3"]()
|
||||
else:
|
||||
self.text_config = text_config
|
||||
|
||||
self.downsample_ratio = downsample_ratio
|
||||
self.dynamic_image_size = dynamic_image_size
|
||||
self.force_image_size = force_image_size
|
||||
self.image_token_id = image_token_id
|
||||
self.max_dynamic_patch = max_dynamic_patch
|
||||
self.min_dynamic_patch = min_dynamic_patch
|
||||
self.pad2square = pad2square
|
||||
self.ps_version = ps_version
|
||||
self.select_layer = select_layer
|
||||
self.template = template
|
||||
self.use_thumbnail = use_thumbnail
|
||||
self.tie_word_embeddings = tie_word_embeddings
|
||||
Reference in New Issue
Block a user