[Model] Support MiniCPM-V 4.7 (#58674)
Signed-off-by: tc-mb <tianchi_cai@icloud.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -45,6 +45,7 @@ from .interfaces import (
|
||||
HasInnerState,
|
||||
IsHybrid,
|
||||
MultiModalEmbeddings,
|
||||
SupportsEagle3,
|
||||
SupportsLoRA,
|
||||
SupportsMRoPE,
|
||||
SupportsMultiModal,
|
||||
@@ -87,35 +88,186 @@ def _minicpmv4_6_field_config(hf_inputs: Mapping[str, torch.Tensor]):
|
||||
fields["use_vit_merger"] = MultiModalFieldConfig.batched(
|
||||
"image", keep_on_cpu=True
|
||||
)
|
||||
if "video_use_vit_merger" in hf_inputs:
|
||||
fields["video_use_vit_merger"] = MultiModalFieldConfig.batched(
|
||||
"video", keep_on_cpu=True
|
||||
)
|
||||
return fields
|
||||
|
||||
|
||||
def _expand_flags(value: object, num_items: int) -> list[bool | None]:
|
||||
"""Broadcast a batched per-item flag to exactly one entry per item."""
|
||||
if value is None:
|
||||
return [None] * num_items
|
||||
|
||||
raw: list[object]
|
||||
if isinstance(value, torch.Tensor):
|
||||
raw = value.reshape(-1).tolist()
|
||||
elif isinstance(value, list | tuple):
|
||||
raw = []
|
||||
for item in value:
|
||||
if isinstance(item, torch.Tensor):
|
||||
raw.extend(item.reshape(-1).tolist())
|
||||
else:
|
||||
raw.append(item)
|
||||
else:
|
||||
raw = [value]
|
||||
|
||||
if len(raw) == num_items:
|
||||
return [bool(v) for v in raw]
|
||||
if len(raw) == 1:
|
||||
return [bool(raw[0])] * num_items
|
||||
# Unexpected shape: keep one decision for the whole batch instead of
|
||||
# failing the request.
|
||||
return [any(bool(v) for v in raw)] * num_items
|
||||
|
||||
|
||||
def _vision_downsample_mode(flag: bool | None) -> str | None:
|
||||
if flag is None:
|
||||
return None
|
||||
return "16x" if flag else "4x"
|
||||
|
||||
|
||||
def _select_vision_items(
|
||||
image_input: MiniCPMVImagePixelInputs,
|
||||
item_indices: list[int],
|
||||
num_slices: list[int],
|
||||
) -> MiniCPMVImagePixelInputs:
|
||||
"""Slice a vision input down to the given items, keeping their order."""
|
||||
offsets = [0]
|
||||
for n in num_slices:
|
||||
offsets.append(offsets[-1] + n)
|
||||
|
||||
pixel_values: list[torch.Tensor] = []
|
||||
tgt_sizes: list[torch.Tensor] = []
|
||||
sizes: list[int] = []
|
||||
for i in item_indices:
|
||||
start, end = offsets[i], offsets[i + 1]
|
||||
pixel_values.extend(image_input["pixel_values"][start:end])
|
||||
tgt_sizes.append(image_input["tgt_sizes"][start:end])
|
||||
sizes.append(end - start)
|
||||
|
||||
return MiniCPMVImagePixelInputs(
|
||||
type="pixel_values",
|
||||
pixel_values=pixel_values,
|
||||
tgt_sizes=torch.cat(tgt_sizes, dim=0),
|
||||
num_slices=torch.tensor(sizes),
|
||||
)
|
||||
|
||||
|
||||
# HF-style modality-scoped keys inside `mm_processor_kwargs`. They scope a value
|
||||
# to a single modality and must not be forwarded to the processors, which take
|
||||
# flat arguments only.
|
||||
_MODALITY_SCOPED_MM_KWARGS = ("images_kwargs", "videos_kwargs", "audio_kwargs")
|
||||
|
||||
|
||||
def _flat_processor_kwargs(
|
||||
mm_kwargs: Mapping[str, object],
|
||||
downsample_mode: str,
|
||||
) -> dict[str, object]:
|
||||
"""Drop the modality-scoped nesting and pin the resolved downsample mode.
|
||||
|
||||
The placeholder count and the vision tower must read the same mode, so the
|
||||
value resolved for this modality is passed down explicitly rather than
|
||||
leaving the processor to fall back to its own default.
|
||||
"""
|
||||
kwargs = {
|
||||
key: value
|
||||
for key, value in mm_kwargs.items()
|
||||
if key not in _MODALITY_SCOPED_MM_KWARGS
|
||||
}
|
||||
kwargs["downsample_mode"] = downsample_mode
|
||||
return kwargs
|
||||
|
||||
|
||||
_MODALITY_TO_MM_KWARGS_SCOPE = {
|
||||
"image": "images_kwargs",
|
||||
"video": "videos_kwargs",
|
||||
}
|
||||
|
||||
|
||||
def _scoped_mm_kwarg(
|
||||
merged: Mapping[str, object],
|
||||
modality: str,
|
||||
key: str,
|
||||
) -> Any:
|
||||
"""Read a processor kwarg for one modality.
|
||||
|
||||
``merged`` is what ``_merge_and_resolve_mm_processor_kwargs`` returns: it
|
||||
routes a flat key into every scope whose processor declares it, and a value
|
||||
scoped to the other modality stays there, so it is not visible here.
|
||||
"""
|
||||
scoped = merged.get(_MODALITY_TO_MM_KWARGS_SCOPE[modality])
|
||||
if isinstance(scoped, Mapping):
|
||||
return scoped.get(key)
|
||||
return None
|
||||
|
||||
|
||||
class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
def _resolve_downsample_mode(
|
||||
self,
|
||||
mm_kwargs: Mapping[str, object],
|
||||
modality: str = "image",
|
||||
) -> str:
|
||||
ds = mm_kwargs.get("downsample_mode")
|
||||
if ds is not None:
|
||||
return str(ds)
|
||||
# A flat `downsample_mode` applies to both modalities; the nested
|
||||
# `images_kwargs` / `videos_kwargs` forms scope it to one, matching
|
||||
# MiniCPMV4_6Processor.
|
||||
info = self.info
|
||||
assert isinstance(info, MiniCPMV4_6ProcessingInfo)
|
||||
merged = info._merge_and_resolve_mm_processor_kwargs(mm_kwargs)
|
||||
downsample_mode = _scoped_mm_kwarg(merged, modality, "downsample_mode")
|
||||
if downsample_mode is not None:
|
||||
return str(downsample_mode)
|
||||
return info._get_downsample_mode()
|
||||
|
||||
def _resolve_max_slice_nums(
|
||||
self,
|
||||
mm_kwargs: Mapping[str, object],
|
||||
modality: str = "image",
|
||||
) -> int | None:
|
||||
# Per-request max_slice_nums. None keeps the processor default.
|
||||
# Resolved like `downsample_mode` so the nested per-modality form
|
||||
# (`images_kwargs` / `videos_kwargs`) is honoured too. This must match
|
||||
# what `process_images` passes to the image processor: the placeholder
|
||||
# count and the visual token count are derived from it separately, so
|
||||
# they diverge (and the engine aborts) if the two sides read different
|
||||
# values.
|
||||
info = self.info
|
||||
assert isinstance(info, MiniCPMV4_6ProcessingInfo)
|
||||
merged = info._merge_and_resolve_mm_processor_kwargs(mm_kwargs)
|
||||
max_slice = _scoped_mm_kwarg(merged, modality, "max_slice_nums")
|
||||
if max_slice is None:
|
||||
return None
|
||||
return int(max_slice)
|
||||
|
||||
def get_image_prompt_texts(
|
||||
self,
|
||||
image_size,
|
||||
image_idx: int = 0,
|
||||
downsample_mode: str | None = None,
|
||||
max_slice_nums: int | None = None,
|
||||
) -> str:
|
||||
info = self.info
|
||||
assert isinstance(info, MiniCPMV4_6ProcessingInfo)
|
||||
return info.get_slice_image_placeholder(
|
||||
image_size,
|
||||
image_idx=image_idx,
|
||||
max_slice_nums=max_slice_nums,
|
||||
downsample_mode=downsample_mode,
|
||||
)
|
||||
|
||||
def _video_local_id_prefix(self, video_idx: int) -> str:
|
||||
"""`<image_id>{idx}</image_id>` prefix emitted in front of a video placeholder.
|
||||
|
||||
MiniCPM-V 4.6 numbers each video with a local id, matching
|
||||
MiniCPMV4_6Processor. 4.7 treats a video as one temporal sequence and is
|
||||
trained without these ids, so it overrides this to return an empty string.
|
||||
"""
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
id_start = getattr(tokenizer, "image_id_start_token", "<image_id>")
|
||||
id_end = getattr(tokenizer, "image_id_end_token", "</image_id>")
|
||||
return f"{id_start}{video_idx}{id_end}"
|
||||
|
||||
def get_video_prompt_texts(
|
||||
self,
|
||||
image_size,
|
||||
@@ -144,8 +296,6 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
image_end = getattr(tokenizer, "image_end_token", "</image>")
|
||||
slice_start = getattr(tokenizer, "slice_start_token", "<slice>")
|
||||
slice_end = getattr(tokenizer, "slice_end_token", "</slice>")
|
||||
id_start = getattr(tokenizer, "image_id_start_token", "<image_id>")
|
||||
id_end = getattr(tokenizer, "image_id_end_token", "</image_id>")
|
||||
|
||||
per_frame = image_start + video_token * source_tokens + image_end
|
||||
if grids[0] > 0 and grids[1] > 0 and patch_tokens > 0:
|
||||
@@ -154,7 +304,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
per_frame += "\n".join(rows)
|
||||
|
||||
body = per_frame * num_frames
|
||||
return f"{id_start}{video_idx}{id_end}" + body
|
||||
return self._video_local_id_prefix(video_idx) + body
|
||||
|
||||
def process_images(
|
||||
self,
|
||||
@@ -179,10 +329,18 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
n_images = len(parsed_images)
|
||||
image_processor = self.info.get_image_processor()
|
||||
patch_size = image_processor.patch_size
|
||||
ds_mode = self._resolve_downsample_mode(mm_kwargs, modality="image")
|
||||
image_proc_kwargs = _flat_processor_kwargs(mm_kwargs, ds_mode)
|
||||
# `_flat_processor_kwargs` drops the nested form, so the value resolved
|
||||
# from it has to be pinned back for the processor and the prompt text
|
||||
# to agree on the slice count.
|
||||
image_max_slice = self._resolve_max_slice_nums(mm_kwargs, modality="image")
|
||||
if image_max_slice is not None:
|
||||
image_proc_kwargs["max_slice_nums"] = image_max_slice
|
||||
per_image_pixel_values: list[torch.Tensor] = []
|
||||
per_image_tgt_sizes: list[torch.Tensor] = []
|
||||
for image in parsed_images:
|
||||
ip_out = image_processor([image], **mm_kwargs)
|
||||
ip_out = image_processor([image], **image_proc_kwargs)
|
||||
pv = ip_out["pixel_values"] # (1, C, P, sum_W)
|
||||
ts = ip_out["target_sizes"] # (n_slices, 2)
|
||||
if pv.ndim == 4 and pv.shape[0] == 1:
|
||||
@@ -210,7 +368,6 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
"tgt_sizes": per_image_tgt_sizes,
|
||||
}
|
||||
|
||||
ds_mode = self._resolve_downsample_mode(mm_kwargs)
|
||||
insert_layer_id = getattr(
|
||||
self.info.get_hf_config(),
|
||||
"insert_layer_id",
|
||||
@@ -245,7 +402,11 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
image_processor = self.info.get_image_processor()
|
||||
patch_size = image_processor.patch_size
|
||||
video_max_slice = self.info.get_video_max_slice_num()
|
||||
video_mm_kwargs = {**mm_kwargs, "max_slice_nums": video_max_slice}
|
||||
video_ds_mode = self._resolve_downsample_mode(mm_kwargs, modality="video")
|
||||
video_mm_kwargs = {
|
||||
**_flat_processor_kwargs(mm_kwargs, video_ds_mode),
|
||||
"max_slice_nums": video_max_slice,
|
||||
}
|
||||
|
||||
per_video_pixel_values: list[torch.Tensor] = []
|
||||
per_video_tgt_sizes: list[torch.Tensor] = []
|
||||
@@ -312,10 +473,17 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
if not per_video_pixel_values:
|
||||
return {}
|
||||
|
||||
insert_layer_id = getattr(self.info.get_hf_config(), "insert_layer_id", -1)
|
||||
merger_flag = video_ds_mode != "4x" and insert_layer_id >= 0
|
||||
|
||||
return {
|
||||
"video_pixel_values": per_video_pixel_values,
|
||||
"video_tgt_sizes": per_video_tgt_sizes,
|
||||
"video_image_sizes": per_video_image_sizes,
|
||||
"video_use_vit_merger": [
|
||||
torch.tensor([merger_flag], dtype=torch.bool)
|
||||
for _ in per_video_pixel_values
|
||||
],
|
||||
}
|
||||
|
||||
def _get_prompt_updates(
|
||||
@@ -324,7 +492,13 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
out_mm_kwargs,
|
||||
):
|
||||
ds_mode = self._resolve_downsample_mode(hf_processor_mm_kwargs)
|
||||
image_ds_mode = self._resolve_downsample_mode(
|
||||
hf_processor_mm_kwargs, modality="image"
|
||||
)
|
||||
video_ds_mode = self._resolve_downsample_mode(
|
||||
hf_processor_mm_kwargs, modality="video"
|
||||
)
|
||||
image_max_slice = self._resolve_max_slice_nums(hf_processor_mm_kwargs)
|
||||
|
||||
placeholders = [
|
||||
("image", self.info.image_pattern),
|
||||
@@ -360,7 +534,8 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
self.get_image_prompt_texts(
|
||||
image_size,
|
||||
item_idx,
|
||||
downsample_mode=ds_mode,
|
||||
downsample_mode=image_ds_mode,
|
||||
max_slice_nums=image_max_slice,
|
||||
),
|
||||
add_special_tokens=False,
|
||||
),
|
||||
@@ -389,7 +564,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
self.get_video_prompt_texts(
|
||||
frame_size,
|
||||
num_frames,
|
||||
downsample_mode=ds_mode,
|
||||
downsample_mode=video_ds_mode,
|
||||
video_idx=item_idx,
|
||||
),
|
||||
add_special_tokens=False,
|
||||
@@ -409,7 +584,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
self.get_video_prompt_texts(
|
||||
frame_size,
|
||||
num_frames,
|
||||
downsample_mode=ds_mode,
|
||||
downsample_mode=video_ds_mode,
|
||||
video_idx=item_idx,
|
||||
),
|
||||
add_special_tokens=False,
|
||||
@@ -437,31 +612,28 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
|
||||
new_update = super()._recompute_cached_prompt_update(
|
||||
cached_update, new_item_idx
|
||||
)
|
||||
# MiniCPM-V 4.6 prefixes video placeholders with `<image_id>{idx}</image_id>`
|
||||
# (the base class only rewrites the image modality).
|
||||
# Video placeholders carry an `<image_id>{idx}</image_id>` prefix that
|
||||
# the base class does not rewrite. 4.7 has no prefix, so this is a
|
||||
# no-op there.
|
||||
if cached_update.modality == "video":
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
id_start = getattr(tokenizer, "image_id_start_token", "<image_id>")
|
||||
id_end = getattr(tokenizer, "image_id_end_token", "</image_id>")
|
||||
video_token = getattr(tokenizer, "video_token", "<|video_pad|>")
|
||||
old_prefix = self._video_local_id_prefix(cached_update.item_idx)
|
||||
new_prefix = self._video_local_id_prefix(new_item_idx)
|
||||
if old_prefix != new_prefix:
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
video_token = getattr(tokenizer, "video_token", "<|video_pad|>")
|
||||
|
||||
text = tokenizer.decode(cached_update.content.full)
|
||||
prev_item_idx = cached_update.item_idx
|
||||
text = tokenizer.decode(cached_update.content.full)
|
||||
|
||||
new_update = new_update.with_content(
|
||||
PromptUpdateDetails.select_token_ids(
|
||||
cached_encode(
|
||||
tokenizer,
|
||||
text.replace(
|
||||
f"{id_start}{prev_item_idx}{id_end}",
|
||||
f"{id_start}{new_item_idx}{id_end}",
|
||||
1,
|
||||
new_update = new_update.with_content(
|
||||
PromptUpdateDetails.select_token_ids(
|
||||
cached_encode(
|
||||
tokenizer,
|
||||
text.replace(old_prefix, new_prefix, 1),
|
||||
add_special_tokens=False,
|
||||
),
|
||||
add_special_tokens=False,
|
||||
),
|
||||
cached_encode(tokenizer, video_token, add_special_tokens=False),
|
||||
cached_encode(tokenizer, video_token, add_special_tokens=False),
|
||||
)
|
||||
)
|
||||
)
|
||||
return new_update
|
||||
|
||||
def _get_mm_fields_config(
|
||||
@@ -676,6 +848,31 @@ class MiniCPMV4_6ProcessingInfo(MiniCPMVProcessingInfo):
|
||||
return source_tokens + grids[0] * grids[1] * patch_tokens
|
||||
|
||||
|
||||
def _stack_vit_merger_qkv(
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
"""Fold separate vit_merger q/k/v projections into the fused qkv_proj module.
|
||||
|
||||
HF checkpoints store the window-attention projections separately, while
|
||||
vLLM keeps them fused. The shard_id mechanism on the submodule's
|
||||
WeightsMapper cannot do this: AutoWeightsLoader still recurses over every
|
||||
incoming key after calling a submodule's load_weights, so a "k_proj" key
|
||||
with no matching child raises before the mapping is honoured.
|
||||
"""
|
||||
pairs = (
|
||||
("vit_merger.self_attn.q_proj.", "q"),
|
||||
("vit_merger.self_attn.k_proj.", "k"),
|
||||
("vit_merger.self_attn.v_proj.", "v"),
|
||||
)
|
||||
for name, tensor in weights:
|
||||
for src, shard in pairs:
|
||||
if src in name:
|
||||
name = name.replace(src, "vit_merger.self_attn.qkv_proj.", 1)
|
||||
tensor.shard_id = shard
|
||||
break
|
||||
yield name, tensor
|
||||
|
||||
|
||||
class MiniCPMV4_6ViTWindowAttentionSelfAttn(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -957,6 +1154,7 @@ class MiniCPMV4_6ForConditionalGeneration(
|
||||
SupportsPP,
|
||||
HasInnerState,
|
||||
IsHybrid,
|
||||
SupportsEagle3,
|
||||
SupportsMRoPE,
|
||||
):
|
||||
supports_encoder_tp_data = True
|
||||
@@ -1031,7 +1229,7 @@ class MiniCPMV4_6ForConditionalGeneration(
|
||||
apply_encoder_attention_mask=True,
|
||||
prefix=maybe_prefix(prefix, "vpm"),
|
||||
)
|
||||
if config.drop_vision_last_layer:
|
||||
if self._drop_vision_last_layer():
|
||||
self.vpm.encoder.layers = self.vpm.encoder.layers[:-1]
|
||||
|
||||
self.vit_merger = MiniCPMV4_6ViTWindowAttentionMerger(
|
||||
@@ -1062,6 +1260,14 @@ class MiniCPMV4_6ForConditionalGeneration(
|
||||
self.language_model.make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
def _drop_vision_last_layer(self) -> bool:
|
||||
"""Whether the last vision encoder layer is dropped.
|
||||
|
||||
MiniCPM-V 4.6 checkpoints always define this. 4.7's config no longer
|
||||
carries the field, so it overrides this.
|
||||
"""
|
||||
return self.config.drop_vision_last_layer
|
||||
|
||||
# ----- Multimodal parsing -----
|
||||
|
||||
def _parse_and_validate_vision_input(
|
||||
@@ -1174,35 +1380,35 @@ class MiniCPMV4_6ForConditionalGeneration(
|
||||
if image_input["type"] == "image_embeds":
|
||||
return image_input["image_embeds"]
|
||||
|
||||
downsample_mode = None
|
||||
if use_vit_merger is not None:
|
||||
downsample_mode = "16x" if use_vit_merger else "4x"
|
||||
image_features = self.get_vision_hidden_states(
|
||||
image_input,
|
||||
downsample_mode=downsample_mode,
|
||||
)
|
||||
num_slices = image_input["num_slices"]
|
||||
results = []
|
||||
idx = 0
|
||||
for n in num_slices.tolist():
|
||||
group = image_features[idx : idx + n]
|
||||
results.append(torch.cat(group, dim=0))
|
||||
idx += n
|
||||
return results
|
||||
num_slices = image_input["num_slices"].tolist()
|
||||
flags = _expand_flags(use_vit_merger, len(num_slices))
|
||||
|
||||
# The merger flag is per request, but the model runner may batch items
|
||||
# from several concurrent requests. Encoding such a mixed batch would
|
||||
# apply one request's downsample mode to another request's items, so
|
||||
# group by flag and encode each group separately.
|
||||
encoded: dict[int, torch.Tensor] = {}
|
||||
for flag in dict.fromkeys(flags):
|
||||
item_indices = [i for i, f in enumerate(flags) if f == flag]
|
||||
group_input = _select_vision_items(image_input, item_indices, num_slices)
|
||||
group_features = self.get_vision_hidden_states(
|
||||
group_input,
|
||||
downsample_mode=_vision_downsample_mode(flag),
|
||||
)
|
||||
|
||||
idx = 0
|
||||
for i in item_indices:
|
||||
n = num_slices[i]
|
||||
encoded[i] = torch.cat(group_features[idx : idx + n], dim=0)
|
||||
idx += n
|
||||
|
||||
return [encoded[i] for i in range(len(num_slices))]
|
||||
|
||||
# ----- Multimodal embedding interface -----
|
||||
|
||||
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
|
||||
use_vit_merger_tensors = kwargs.pop("use_vit_merger", None)
|
||||
use_vit_merger = None
|
||||
if use_vit_merger_tensors is not None:
|
||||
if isinstance(use_vit_merger_tensors, torch.Tensor):
|
||||
use_vit_merger = bool(use_vit_merger_tensors.any().item())
|
||||
elif isinstance(use_vit_merger_tensors, list | tuple):
|
||||
use_vit_merger = any(
|
||||
bool(t.any().item()) if isinstance(t, torch.Tensor) else bool(t)
|
||||
for t in use_vit_merger_tensors
|
||||
)
|
||||
use_vit_merger = kwargs.pop("use_vit_merger", None)
|
||||
video_use_vit_merger = kwargs.pop("video_use_vit_merger", None)
|
||||
|
||||
image_kwargs = {
|
||||
k: v
|
||||
@@ -1235,7 +1441,7 @@ class MiniCPMV4_6ForConditionalGeneration(
|
||||
multimodal_embeddings += tuple(
|
||||
self._process_vision_input(
|
||||
video_input,
|
||||
use_vit_merger=use_vit_merger,
|
||||
use_vit_merger=video_use_vit_merger,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -1298,7 +1504,10 @@ class MiniCPMV4_6ForConditionalGeneration(
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
return loader.load_weights(
|
||||
_stack_vit_merger_qkv(weights),
|
||||
mapper=self.hf_to_vllm_mapper,
|
||||
)
|
||||
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
return MultiModelKeys.from_string_field(
|
||||
|
||||
Reference in New Issue
Block a user