[Bugfix] MiniCPM-V 4.6: fix ViT self-attn qkv weight loading (#49417)
Signed-off-by: arkohut <39525455+arkohut@users.noreply.github.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -674,14 +674,6 @@ class MiniCPMV4_6ProcessingInfo(MiniCPMVProcessingInfo):
|
||||
|
||||
|
||||
class MiniCPMV4_6ViTWindowAttentionSelfAttn(nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config,
|
||||
@@ -729,10 +721,6 @@ class MiniCPMV4_6ViTWindowAttentionSelfAttn(nn.Module):
|
||||
out, _ = self.out_proj(attn_out)
|
||||
return out
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class MiniCPMV4_6ViTWindowAttentionMerger(nn.Module):
|
||||
def __init__(
|
||||
@@ -971,6 +959,11 @@ class MiniCPMV4_6ForConditionalGeneration(
|
||||
supports_encoder_tp_data = True
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
},
|
||||
orig_to_new_prefix={
|
||||
# transformers v5.7+ uses `vision_tower` and nests `vit_merger`
|
||||
# inside it. Order matters: more specific prefix must come first.
|
||||
@@ -982,7 +975,7 @@ class MiniCPMV4_6ForConditionalGeneration(
|
||||
"model.language_model.": "language_model.model.",
|
||||
"lm_head.": "language_model.lm_head.",
|
||||
"mtp.": None,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
packed_modules_mapping = {
|
||||
|
||||
Reference in New Issue
Block a user