[Model] Optimize Sarvam MLA routing and preserve FP32 router logits (#56034)

Signed-off-by: Harshit Kedia <harshit@sarvam.ai>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harshit Kedia
2026-09-16 20:02:47 +08:00
committed by GitHub
co-authored by Claude Opus 5
parent 3bb7826214
commit 6a2fdf9ac6
+21 -10
View File
@@ -36,7 +36,11 @@ from vllm.distributed import (
get_tensor_model_parallel_world_size,
)
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.fused_moe import FusedMoEFactory, MoERunner
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
MoERunner,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -306,11 +310,11 @@ class SarvamMLAMoE(nn.Module):
else:
self.router_dtype = torch.bfloat16
self.gate = nn.Linear(
self.gate = GateLinear(
self.hidden_size,
self.num_experts,
bias=False,
dtype=self.router_dtype,
out_dtype=self.router_dtype,
prefix=f"{prefix}.gate",
)
if getattr(config, "moe_router_enable_expert_bias", True):
@@ -365,12 +369,7 @@ class SarvamMLAMoE(nn.Module):
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
num_tokens, hidden_dim = hidden_states.shape
hidden_states = hidden_states.view(-1, hidden_dim)
router_logits = self.gate(
hidden_states.to(self.router_dtype)
if self.router_dtype is not None
else hidden_states
)
router_logits = router_logits.to(hidden_states.dtype)
router_logits, _ = self.gate(hidden_states)
final_hidden = self.experts(
hidden_states=hidden_states,
router_logits=router_logits,
@@ -643,9 +642,21 @@ class SarvamMLAForCausalLM(
"gate_up_proj": ["gate_proj", "up_proj"],
}
@staticmethod
def _remap_config(config) -> None:
"""Default the routing keys the released checkpoints omit."""
defaults = {
"n_group": 1,
"topk_group": 1,
}
for attr, default in defaults.items():
if getattr(config, attr, None) is None:
setattr(config, attr, default)
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
config = vllm_config.model_config.hf_config
self._remap_config(config)
quant_config = vllm_config.quant_config
self.config = config
self.quant_config = quant_config