[Model] support Qianfan-OCR model (#40136)

Signed-off-by: bairongz <baiyuu.cs@gmail.com>
Signed-off-by: zhuangbairong <zhuangbairong@baidu.com>
Co-authored-by: zhuangbairong <zhuangbairong@baidu.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
bairongz
2026-05-05 10:51:25 +00:00
committed by GitHub
co-authored by zhuangbairong gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent 8b9ea2f881
commit 0a201b60cf
9 changed files with 312 additions and 0 deletions
@@ -928,6 +928,16 @@ VLM_TEST_SETTINGS = {
),
],
),
"qianfan_ocr": VLMTestInfo(
models=["baidu/Qianfan-OCR"],
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
prompt_formatter=lambda img_prompt: f"<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
img_idx_to_prompt=lambda idx: "<image>",
max_model_len=4096,
use_tokenizer_eos=True,
auto_cls=AutoModelForImageTextToText,
hf_model_kwargs=model_utils.qianfan_ocr_hf_model_kwargs("baidu/Qianfan-OCR"),
),
"qwen_vl": VLMTestInfo(
models=["Qwen/Qwen-VL"],
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
@@ -1554,3 +1554,94 @@ def moondream3_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
hf_model.model.generate = types.MethodType(_generate, hf_model.model)
return hf_model
def qianfan_ocr_hf_model_kwargs(model_name: str) -> dict:
"""Return hf_model_kwargs with a patched config for QianfanOCR."""
from vllm.transformers_utils.configs.qianfan_ocr import QianfanOCRConfig
config = QianfanOCRConfig.from_pretrained(model_name)
vc = config.vision_config
if isinstance(vc.image_size, int):
vc.image_size = (vc.image_size, vc.image_size)
if isinstance(vc.patch_size, int):
vc.patch_size = (vc.patch_size, vc.patch_size)
return {"config": config}
def qianfan_ocr_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
"""Patches an HfRunner instance to run QianfanOCR model inference.
QianfanOCR shares the same architecture as InternVLChatModel, so the
patching logic mirrors ``internvl_patch_hf_runner``. The only difference
is that we load the config via vllm's registered ``QianfanOCRConfig``
instead of relying on ``trust_remote_code``.
"""
class QianfanOCRProcessor:
def __init__(self, hf_runner: HfRunner):
self.tokenizer = hf_runner.tokenizer
from vllm.transformers_utils.configs.qianfan_ocr import QianfanOCRConfig
self.config = QianfanOCRConfig.from_pretrained(hf_runner.model_name)
self.vision_config = self.config.vision_config
self.use_thumbnail = self.config.use_thumbnail
self.min_num = self.config.min_dynamic_patch
self.max_num = self.config.max_dynamic_patch
self.image_size = self.vision_config.image_size
# Compute num_image_token from config instead of model attribute,
# since the transformers-native model doesn't expose it.
image_size = self.config.force_image_size or self.vision_config.image_size
patch_size = self.vision_config.patch_size
downsample_ratio = self.config.downsample_ratio
self.num_image_token = int(
(image_size // patch_size) ** 2 * (downsample_ratio**2)
)
def __call__(
self,
text: str,
images: PIL.Image.Image | list[PIL.Image.Image] = None,
**kwargs,
):
from vllm.transformers_utils.processors.internvl import (
image_to_pixel_values_internvl,
)
IMG_START = "<img>"
IMG_END = "</img>"
IMG_CONTEXT = "<IMG_CONTEXT>"
images = [images] if isinstance(images, PIL.Image.Image) else images
pixel_values_list = [
image_to_pixel_values_internvl(
image,
input_size=self.image_size,
min_num=self.min_num,
max_num=self.max_num,
use_thumbnail=self.use_thumbnail,
)
for image in images
]
num_patches_list = [pv.shape[0] for pv in pixel_values_list]
pixel_values = torch.cat(pixel_values_list, dim=0)
for num_patches in num_patches_list:
context_tokens = IMG_CONTEXT * self.num_image_token * num_patches
image_tokens = IMG_START + context_tokens + IMG_END
text = text.replace("<image>", image_tokens, 1)
prompt = self.tokenizer(text, return_tensors="pt")
prompt.update({"pixel_values": pixel_values})
return prompt
img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")
hf_model.model.img_context_token_id = img_context_token_id
hf_model.processor = QianfanOCRProcessor(hf_model)
hf_model.model.get_output_embeddings = (
lambda: hf_model.model.language_model.get_output_embeddings()
)
hf_model.model.generate = types.MethodType(_internvl_generate, hf_model.model)
return hf_model
+4
View File
@@ -1264,6 +1264,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
},
tokenizer_mode="mistral",
),
"QianfanOCRForConditionalGeneration": _HfExamplesInfo(
"baidu/Qianfan-OCR",
min_transformers_version="5.6.0",
),
"QwenVLForConditionalGeneration": _HfExamplesInfo(
"Qwen/Qwen-VL",
extras={"chat": "Qwen/Qwen-VL-Chat"},