[Bugfix][LoRA] Add multimodal module mapping for Muse-Glimmer (#53513)

Signed-off-by: linitra24 <Joy25810@foxmail.com>
This commit is contained in:
linitra24
2026-08-24 05:51:43 +00:00
committed by GitHub
parent 460c08bc8a
commit 8c2bbe00d5
@@ -69,6 +69,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
@@ -1646,3 +1647,13 @@ class MuseGlimmerForCausalLM(
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def get_mm_mapping(self) -> MultiModelKeys:
"""
Get the module prefix in multimodal models
"""
return MultiModelKeys.from_string_field(
language_model="model",
connector=["vision_adapter.", "vision_projection."],
tower_model="vision_encoder.",
)