[Models] Cohere MoE (#40817)

Signed-off-by: Terrencezzj <terrence@cohere.ai>
This commit is contained in:
Terrence Zhao
2026-04-29 15:54:54 +00:00
committed by GitHub
parent a05848e255
commit 91a2d39014
7 changed files with 554 additions and 3 deletions
+14 -3
View File
@@ -44,7 +44,12 @@ from vllm.multimodal.processing import (
from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
from .interfaces import (
MultiModalEmbeddings,
SupportsMultiModal,
SupportsPP,
SupportsQuant,
)
from .siglip import SiglipVisionModel
from .utils import (
AutoWeightsLoader,
@@ -309,16 +314,22 @@ class Cohere2VisionMultiModalProcessor(
info=Cohere2VisionProcessingInfo,
dummy_inputs=Cohere2VisionDummyInputsBuilder,
)
class Cohere2VisionForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
class Cohere2VisionForConditionalGeneration(
nn.Module, SupportsMultiModal, SupportsPP, SupportsQuant
):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={
"model.vision_tower.": "vision_tower.",
"model.multi_modal_projector.": "multi_modal_projector.",
"model.language_model.": "language_model.model.",
"lm_head.": "language_model.lm_head.",
}
)
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
"gate_up_proj": ["gate_proj", "up_proj"],
}
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config: Cohere2VisionConfig = vllm_config.model_config.hf_config