diff --git a/vllm/model_executor/models/sarvam.py b/vllm/model_executor/models/sarvam.py index 14d68fec56..2cc8a27f4d 100644 --- a/vllm/model_executor/models/sarvam.py +++ b/vllm/model_executor/models/sarvam.py @@ -36,7 +36,11 @@ from vllm.distributed import ( get_tensor_model_parallel_world_size, ) from vllm.model_executor.layers.activation import SiluAndMul -from vllm.model_executor.layers.fused_moe import FusedMoEFactory, MoERunner +from vllm.model_executor.layers.fused_moe import ( + FusedMoEFactory, + GateLinear, + MoERunner, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -306,11 +310,11 @@ class SarvamMLAMoE(nn.Module): else: self.router_dtype = torch.bfloat16 - self.gate = nn.Linear( + self.gate = GateLinear( self.hidden_size, self.num_experts, - bias=False, - dtype=self.router_dtype, + out_dtype=self.router_dtype, + prefix=f"{prefix}.gate", ) if getattr(config, "moe_router_enable_expert_bias", True): @@ -365,12 +369,7 @@ class SarvamMLAMoE(nn.Module): def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: num_tokens, hidden_dim = hidden_states.shape hidden_states = hidden_states.view(-1, hidden_dim) - router_logits = self.gate( - hidden_states.to(self.router_dtype) - if self.router_dtype is not None - else hidden_states - ) - router_logits = router_logits.to(hidden_states.dtype) + router_logits, _ = self.gate(hidden_states) final_hidden = self.experts( hidden_states=hidden_states, router_logits=router_logits, @@ -643,9 +642,21 @@ class SarvamMLAForCausalLM( "gate_up_proj": ["gate_proj", "up_proj"], } + @staticmethod + def _remap_config(config) -> None: + """Default the routing keys the released checkpoints omit.""" + defaults = { + "n_group": 1, + "topk_group": 1, + } + for attr, default in defaults.items(): + if getattr(config, attr, None) is None: + setattr(config, attr, default) + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: super().__init__() config = vllm_config.model_config.hf_config + self._remap_config(config) quant_config = vllm_config.quant_config self.config = config self.quant_config = quant_config