[Model] support Qianfan-OCR model (#40136)
Signed-off-by: bairongz <baiyuu.cs@gmail.com> Signed-off-by: zhuangbairong <zhuangbairong@baidu.com> Co-authored-by: zhuangbairong <zhuangbairong@baidu.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
zhuangbairong
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent
8b9ea2f881
commit
0a201b60cf
@@ -928,6 +928,16 @@ VLM_TEST_SETTINGS = {
|
||||
),
|
||||
],
|
||||
),
|
||||
"qianfan_ocr": VLMTestInfo(
|
||||
models=["baidu/Qianfan-OCR"],
|
||||
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
|
||||
prompt_formatter=lambda img_prompt: f"<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
|
||||
img_idx_to_prompt=lambda idx: "<image>",
|
||||
max_model_len=4096,
|
||||
use_tokenizer_eos=True,
|
||||
auto_cls=AutoModelForImageTextToText,
|
||||
hf_model_kwargs=model_utils.qianfan_ocr_hf_model_kwargs("baidu/Qianfan-OCR"),
|
||||
),
|
||||
"qwen_vl": VLMTestInfo(
|
||||
models=["Qwen/Qwen-VL"],
|
||||
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
|
||||
|
||||
@@ -1554,3 +1554,94 @@ def moondream3_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||
|
||||
hf_model.model.generate = types.MethodType(_generate, hf_model.model)
|
||||
return hf_model
|
||||
|
||||
|
||||
def qianfan_ocr_hf_model_kwargs(model_name: str) -> dict:
|
||||
"""Return hf_model_kwargs with a patched config for QianfanOCR."""
|
||||
from vllm.transformers_utils.configs.qianfan_ocr import QianfanOCRConfig
|
||||
|
||||
config = QianfanOCRConfig.from_pretrained(model_name)
|
||||
vc = config.vision_config
|
||||
if isinstance(vc.image_size, int):
|
||||
vc.image_size = (vc.image_size, vc.image_size)
|
||||
if isinstance(vc.patch_size, int):
|
||||
vc.patch_size = (vc.patch_size, vc.patch_size)
|
||||
return {"config": config}
|
||||
|
||||
|
||||
def qianfan_ocr_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
|
||||
"""Patches an HfRunner instance to run QianfanOCR model inference.
|
||||
|
||||
QianfanOCR shares the same architecture as InternVLChatModel, so the
|
||||
patching logic mirrors ``internvl_patch_hf_runner``. The only difference
|
||||
is that we load the config via vllm's registered ``QianfanOCRConfig``
|
||||
instead of relying on ``trust_remote_code``.
|
||||
"""
|
||||
|
||||
class QianfanOCRProcessor:
|
||||
def __init__(self, hf_runner: HfRunner):
|
||||
self.tokenizer = hf_runner.tokenizer
|
||||
|
||||
from vllm.transformers_utils.configs.qianfan_ocr import QianfanOCRConfig
|
||||
|
||||
self.config = QianfanOCRConfig.from_pretrained(hf_runner.model_name)
|
||||
self.vision_config = self.config.vision_config
|
||||
self.use_thumbnail = self.config.use_thumbnail
|
||||
self.min_num = self.config.min_dynamic_patch
|
||||
self.max_num = self.config.max_dynamic_patch
|
||||
self.image_size = self.vision_config.image_size
|
||||
|
||||
# Compute num_image_token from config instead of model attribute,
|
||||
# since the transformers-native model doesn't expose it.
|
||||
image_size = self.config.force_image_size or self.vision_config.image_size
|
||||
patch_size = self.vision_config.patch_size
|
||||
downsample_ratio = self.config.downsample_ratio
|
||||
self.num_image_token = int(
|
||||
(image_size // patch_size) ** 2 * (downsample_ratio**2)
|
||||
)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
text: str,
|
||||
images: PIL.Image.Image | list[PIL.Image.Image] = None,
|
||||
**kwargs,
|
||||
):
|
||||
from vllm.transformers_utils.processors.internvl import (
|
||||
image_to_pixel_values_internvl,
|
||||
)
|
||||
|
||||
IMG_START = "<img>"
|
||||
IMG_END = "</img>"
|
||||
IMG_CONTEXT = "<IMG_CONTEXT>"
|
||||
|
||||
images = [images] if isinstance(images, PIL.Image.Image) else images
|
||||
pixel_values_list = [
|
||||
image_to_pixel_values_internvl(
|
||||
image,
|
||||
input_size=self.image_size,
|
||||
min_num=self.min_num,
|
||||
max_num=self.max_num,
|
||||
use_thumbnail=self.use_thumbnail,
|
||||
)
|
||||
for image in images
|
||||
]
|
||||
num_patches_list = [pv.shape[0] for pv in pixel_values_list]
|
||||
pixel_values = torch.cat(pixel_values_list, dim=0)
|
||||
|
||||
for num_patches in num_patches_list:
|
||||
context_tokens = IMG_CONTEXT * self.num_image_token * num_patches
|
||||
image_tokens = IMG_START + context_tokens + IMG_END
|
||||
text = text.replace("<image>", image_tokens, 1)
|
||||
|
||||
prompt = self.tokenizer(text, return_tensors="pt")
|
||||
prompt.update({"pixel_values": pixel_values})
|
||||
return prompt
|
||||
|
||||
img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")
|
||||
hf_model.model.img_context_token_id = img_context_token_id
|
||||
hf_model.processor = QianfanOCRProcessor(hf_model)
|
||||
hf_model.model.get_output_embeddings = (
|
||||
lambda: hf_model.model.language_model.get_output_embeddings()
|
||||
)
|
||||
hf_model.model.generate = types.MethodType(_internvl_generate, hf_model.model)
|
||||
return hf_model
|
||||
|
||||
@@ -1264,6 +1264,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
},
|
||||
tokenizer_mode="mistral",
|
||||
),
|
||||
"QianfanOCRForConditionalGeneration": _HfExamplesInfo(
|
||||
"baidu/Qianfan-OCR",
|
||||
min_transformers_version="5.6.0",
|
||||
),
|
||||
"QwenVLForConditionalGeneration": _HfExamplesInfo(
|
||||
"Qwen/Qwen-VL",
|
||||
extras={"chat": "Qwen/Qwen-VL-Chat"},
|
||||
|
||||
Reference in New Issue
Block a user