[Model] Fix MiniCPM-V 4.6 vit_merger qkv weight loading (#43213)

Signed-off-by: tc-mb <tianchi_cai@icloud.com>
This commit is contained in:
tc-mb
2026-05-21 22:44:06 -07:00
committed by GitHub
parent e746a2eebf
commit fa1ff88b31
+28
View File
@@ -23,6 +23,7 @@ from vllm.model_executor.layers.mamba.mamba_utils import (
MambaStateShapeCalculator,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
@@ -590,6 +591,33 @@ class MiniCPMV4_6ViTWindowAttentionSelfAttn(nn.Module):
out, _ = self.out_proj(attn_out)
return out
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
mapped_name = name.replace(weight_name, param_name, 1)
if mapped_name not in params_dict:
continue
param = params_dict[mapped_name]
param.weight_loader(param, loaded_weight, shard_id)
break
else:
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
class MiniCPMV4_6ViTWindowAttentionMerger(nn.Module):
def __init__(