[Bugfix] MiniCPM-V 4.6: fix ViT self-attn qkv weight loading (#49417)

Signed-off-by: arkohut <39525455+arkohut@users.noreply.github.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
arkohut
2026-09-14 13:18:55 +00:00
committed by GitHub
co-authored by Harry Mellor
parent 47fbd36e6d
commit 6623fe5b4e
+6 -13
View File
@@ -674,14 +674,6 @@ class MiniCPMV4_6ProcessingInfo(MiniCPMVProcessingInfo):
class MiniCPMV4_6ViTWindowAttentionSelfAttn(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
}
)
def __init__(
self,
config,
@@ -729,10 +721,6 @@ class MiniCPMV4_6ViTWindowAttentionSelfAttn(nn.Module):
out, _ = self.out_proj(attn_out)
return out
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class MiniCPMV4_6ViTWindowAttentionMerger(nn.Module):
def __init__(
@@ -971,6 +959,11 @@ class MiniCPMV4_6ForConditionalGeneration(
supports_encoder_tp_data = True
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
},
orig_to_new_prefix={
# transformers v5.7+ uses `vision_tower` and nests `vit_merger`
# inside it. Order matters: more specific prefix must come first.
@@ -982,7 +975,7 @@ class MiniCPMV4_6ForConditionalGeneration(
"model.language_model.": "language_model.model.",
"lm_head.": "language_model.lm_head.",
"mtp.": None,
}
},
)
packed_modules_mapping = {