[Model] Use merge_by_field_config for MM models (M-N) (#26710)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -114,7 +114,7 @@ class MiniCPMVImagePixelInputs(TensorSchema):
|
||||
|
||||
type: Literal["pixel_values"] = "pixel_values"
|
||||
|
||||
# Note that the image size may vary, so we pass it as a list instead of a
|
||||
# Note that the patch size may vary, so we pass it as a list instead of a
|
||||
# batched tensor.
|
||||
pixel_values: Annotated[
|
||||
list[torch.Tensor],
|
||||
@@ -453,12 +453,6 @@ def get_version_by_config(config: PretrainedConfig) -> tuple[int, ...]:
|
||||
|
||||
|
||||
def _minicpmv_field_config(hf_inputs: Mapping[str, torch.Tensor]):
|
||||
pixel_values = hf_inputs.get("pixel_values", torch.empty(0))
|
||||
num_images = len(pixel_values)
|
||||
|
||||
video_pixel_values = hf_inputs.get("video_pixel_values", torch.empty(0))
|
||||
num_videos = len(video_pixel_values)
|
||||
|
||||
return dict(
|
||||
pixel_values=MultiModalFieldConfig.batched("image"),
|
||||
image_sizes=MultiModalFieldConfig.batched("image"),
|
||||
@@ -468,8 +462,6 @@ def _minicpmv_field_config(hf_inputs: Mapping[str, torch.Tensor]):
|
||||
video_image_sizes=MultiModalFieldConfig.batched("video"),
|
||||
video_tgt_sizes=MultiModalFieldConfig.batched("video"),
|
||||
video_embeds=MultiModalFieldConfig.batched("video"),
|
||||
image_token_id=MultiModalFieldConfig.shared("image", num_images),
|
||||
video_token_id=MultiModalFieldConfig.shared("video", num_videos),
|
||||
)
|
||||
|
||||
|
||||
@@ -792,10 +784,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
out_keys={"pixel_values", "image_sizes", "tgt_sizes"},
|
||||
)
|
||||
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
unk_token_id = tokenizer.get_vocab()["<unk>"]
|
||||
image_inputs["image_token_id"] = torch.tensor(unk_token_id)
|
||||
|
||||
return image_inputs
|
||||
|
||||
def process_videos(
|
||||
@@ -831,10 +819,6 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
|
||||
video_inputs = {f"video_{k}": v for k, v in video_inputs.items()}
|
||||
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
unk_token_id = tokenizer.get_vocab()["<unk>"]
|
||||
video_inputs["video_token_id"] = torch.tensor(unk_token_id)
|
||||
|
||||
return video_inputs
|
||||
|
||||
def process_mm_inputs(
|
||||
@@ -1021,6 +1005,8 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
instantiated.
|
||||
"""
|
||||
|
||||
merge_by_field_config = True
|
||||
|
||||
supports_encoder_tp_data = True
|
||||
|
||||
@classmethod
|
||||
@@ -1066,7 +1052,6 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
prefix=maybe_prefix(prefix, "resampler"),
|
||||
)
|
||||
|
||||
self.mm_token_ids = set[int]()
|
||||
self.make_empty_intermediate_tensors = self.llm.make_empty_intermediate_tensors
|
||||
|
||||
def _parse_and_validate_vision_input(
|
||||
@@ -1080,43 +1065,17 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
if pixel_values is None and image_embeds is None:
|
||||
return None
|
||||
|
||||
image_token_id = kwargs.pop("image_token_id")
|
||||
if image_token_id is not None:
|
||||
assert isinstance(image_token_id, torch.Tensor)
|
||||
self.mm_token_ids.add(image_token_id.flatten().unique().item())
|
||||
|
||||
if image_embeds is not None:
|
||||
if not isinstance(image_embeds, (torch.Tensor, list)):
|
||||
raise ValueError(
|
||||
f"Incorrect type of image_embeds for {modality=}. "
|
||||
f"Got type: {type(image_embeds)}"
|
||||
)
|
||||
|
||||
image_embeds_flat = flatten_bn(image_embeds)
|
||||
|
||||
return MiniCPMVImageEmbeddingInputs(
|
||||
type="image_embeds",
|
||||
image_embeds=image_embeds_flat,
|
||||
)
|
||||
|
||||
if not isinstance(pixel_values, (torch.Tensor, list)):
|
||||
raise ValueError(
|
||||
f"Incorrect type of pixel_values for {modality=}. "
|
||||
f"Got type: {type(pixel_values)}"
|
||||
image_embeds=image_embeds,
|
||||
)
|
||||
|
||||
tgt_sizes = kwargs.pop("tgt_sizes")
|
||||
if not isinstance(tgt_sizes, (torch.Tensor, list)):
|
||||
raise ValueError(
|
||||
f"Incorrect type of tgt_sizes for {modality=}. "
|
||||
f"Got type: {type(tgt_sizes)}"
|
||||
)
|
||||
|
||||
num_slices = [[len(p) for p in ps] for ps in pixel_values]
|
||||
num_slices_flat = flatten_bn(torch.tensor(num_slices))
|
||||
|
||||
pixel_values_flat = flatten_bn(flatten_2d_lists(pixel_values))
|
||||
tgt_sizes_flat = flatten_bn(flatten_2d_lists(tgt_sizes), concat=True)
|
||||
num_slices_flat = torch.tensor([len(ps) for ps in pixel_values])
|
||||
pixel_values_flat = flatten_bn(pixel_values)
|
||||
tgt_sizes_flat = flatten_bn(tgt_sizes, concat=True)
|
||||
|
||||
return MiniCPMVImagePixelInputs(
|
||||
type="pixel_values",
|
||||
@@ -1142,15 +1101,8 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
input_key in ("video_pixel_values", "video_embeds")
|
||||
and "videos" not in modalities
|
||||
):
|
||||
|
||||
def _image_key(video_key: str):
|
||||
if video_key == "video_token_id":
|
||||
return "image_token_id"
|
||||
|
||||
return video_key.removeprefix("video_")
|
||||
|
||||
modalities["videos"] = self._parse_and_validate_vision_input(
|
||||
"videos", **{_image_key(k): v for k, v in kwargs.items()}
|
||||
"videos", **{k.removeprefix("video_"): v for k, v in kwargs.items()}
|
||||
)
|
||||
|
||||
return modalities
|
||||
|
||||
Reference in New Issue
Block a user