Support MiniCPMV batched preprocessing (#44609)

Signed-off-by: Yan Ma <yan.ma@intel.com>
This commit is contained in:
Yan Ma
2026-06-05 15:05:31 +00:00
committed by GitHub
parent ef0df7dbd6
commit 7f003a1285
+63 -56
View File
@@ -72,8 +72,8 @@ class MiniCPMVProcessor(ProcessorMixin):
) -> MiniCPMVBatchFeature:
"""Run the vendored MiniCPMV processor on a (text, images) pair.
Only single-sample input is currently supported; batched input is
coming soon. ``images`` is forwarded to the underlying image
Batched inputs are supported following the upstream MiniCPM-V
processor flow. ``images`` is forwarded to the underlying image
processor and ``text`` is tokenized with image placeholders
replaced by the appropriate slice tokens. Returns a
``MiniCPMVBatchFeature`` with at minimum ``input_ids`` and (when
@@ -194,7 +194,7 @@ class MiniCPMVProcessor(ProcessorMixin):
image_end_tokens.unsqueeze(-1),
]
)
return input_ids.unsqueeze(0), image_bounds
return input_ids, image_bounds
def _convert_images_texts_to_inputs(
self,
@@ -220,23 +220,41 @@ class MiniCPMVProcessor(ProcessorMixin):
image_sizes = images["image_sizes"]
tgt_sizes = images["tgt_sizes"]
image_tags = regex.findall(pattern, texts)
assert len(image_tags) == len(image_sizes[0])
text_chunks = texts.split(pattern)
final_texts = ""
for i in range(len(image_tags)):
placeholder = self.image_processor.get_slice_image_placeholder(
image_sizes[0][i]
)
final_texts = final_texts + text_chunks[i] + placeholder
final_texts += text_chunks[-1]
input_ids, image_bounds = self._convert(final_texts, max_length)
if isinstance(texts, str):
texts = [texts]
input_ids_list = []
image_bounds_list = []
for index, text in enumerate(texts):
image_tags = regex.findall(pattern, text)
assert len(image_tags) == len(image_sizes[index])
text_chunks = text.split(pattern)
final_text = ""
for i in range(len(image_tags)):
placeholder = self.image_processor.get_slice_image_placeholder(
image_sizes[index][i]
)
final_text = final_text + text_chunks[i] + placeholder
final_text += text_chunks[-1]
input_ids, image_bounds = self._convert(final_text, max_length)
input_ids_list.append(input_ids)
image_bounds_list.append(image_bounds)
padded_input_ids, padding_lengths = self.pad(
input_ids_list,
padding_side="left",
)
for i, length in enumerate(padding_lengths):
image_bounds_list[i] = image_bounds_list[i] + length
return MiniCPMVBatchFeature(
data={
"input_ids": input_ids,
"input_ids": padded_input_ids,
"attention_mask": padded_input_ids.ne(0),
"pixel_values": images_val,
"image_sizes": image_sizes,
"image_bound": [image_bounds],
"image_bound": image_bounds_list,
"tgt_sizes": tgt_sizes,
}
)
@@ -249,42 +267,36 @@ class MiniCPMVProcessor(ProcessorMixin):
image_processor_input_names = self.image_processor.model_input_names
return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))
def pad(
self,
orig_items,
key,
max_length=None,
padding_value=0,
padding_side="left",
):
if not orig_items:
return torch.empty(0)
# Copied from openbmb/MiniCPM-V-4_5 processing_minicpmv.py.
def pad(self, inputs, max_length=None, padding_value=0, padding_side="left"):
if not inputs:
return torch.empty(0), []
items = []
if isinstance(orig_items[0][key], list):
assert isinstance(orig_items[0][key][0], torch.Tensor)
for it in orig_items:
for tr in it[key]:
items.append({key: tr})
if isinstance(inputs[0], list):
assert isinstance(inputs[0][0], torch.Tensor)
for it in inputs:
for tr in it:
items.append(tr)
else:
assert isinstance(orig_items[0][key], torch.Tensor)
items = orig_items
assert isinstance(inputs[0], torch.Tensor)
items = inputs
batch_size = len(items)
shape = items[0][key].shape
shape = items[0].shape
dim = len(shape)
assert dim <= 3
assert dim <= 2
if max_length is None:
max_length = 0
max_length = max(max_length, max(item[key].shape[-1] for item in items))
min_length = min(item[key].shape[-1] for item in items)
dtype = items[0][key].dtype
max_length = max(max_length, max(item.shape[-1] for item in items))
min_length = min(item.shape[-1] for item in items)
dtype = items[0].dtype
if dim == 1:
return torch.cat([item[key] for item in items], dim=0)
elif dim == 2:
if dim == 0:
return torch.stack([item for item in items], dim=0), [0]
elif dim == 1:
if max_length == min_length:
return torch.cat([item[key] for item in items], dim=0)
return torch.stack([item for item in items], dim=0), [0] * batch_size
tensor = torch.zeros((batch_size, max_length), dtype=dtype) + padding_value
else:
tensor = (
@@ -292,23 +304,18 @@ class MiniCPMVProcessor(ProcessorMixin):
+ padding_value
)
padding_lengths = []
for i, item in enumerate(items):
tensor_to_pad = item[key]
if tensor_to_pad.shape[0] != 1:
raise ValueError(
f"Expected leading batch size of 1 for padding, "
f"but got shape {tensor_to_pad.shape}"
)
squeezed = tensor_to_pad.squeeze(0)
if dim == 2:
if dim == 1:
if padding_side == "left":
tensor[i, -squeezed.shape[0] :] = squeezed.clone()
tensor[i, -len(item) :] = item.clone()
else:
tensor[i, : squeezed.shape[0]] = squeezed.clone()
elif dim == 3:
tensor[i, : len(item)] = item.clone()
elif dim == 2:
if padding_side == "left":
tensor[i, -squeezed.shape[0] :, :] = squeezed.clone()
tensor[i, -len(item) :, :] = item.clone()
else:
tensor[i, : squeezed.shape[0], :] = squeezed.clone()
tensor[i, : len(item), :] = item.clone()
padding_lengths.append(tensor.shape[-1] - len(item))
return tensor
return tensor, padding_lengths