[Model] Optimize Sarvam MLA routing and preserve FP32 router logits (#56034)
Signed-off-by: Harshit Kedia <harshit@sarvam.ai> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
3bb7826214
commit
6a2fdf9ac6
@@ -36,7 +36,11 @@ from vllm.distributed import (
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory, MoERunner
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
GateLinear,
|
||||
MoERunner,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -306,11 +310,11 @@ class SarvamMLAMoE(nn.Module):
|
||||
else:
|
||||
self.router_dtype = torch.bfloat16
|
||||
|
||||
self.gate = nn.Linear(
|
||||
self.gate = GateLinear(
|
||||
self.hidden_size,
|
||||
self.num_experts,
|
||||
bias=False,
|
||||
dtype=self.router_dtype,
|
||||
out_dtype=self.router_dtype,
|
||||
prefix=f"{prefix}.gate",
|
||||
)
|
||||
|
||||
if getattr(config, "moe_router_enable_expert_bias", True):
|
||||
@@ -365,12 +369,7 @@ class SarvamMLAMoE(nn.Module):
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
num_tokens, hidden_dim = hidden_states.shape
|
||||
hidden_states = hidden_states.view(-1, hidden_dim)
|
||||
router_logits = self.gate(
|
||||
hidden_states.to(self.router_dtype)
|
||||
if self.router_dtype is not None
|
||||
else hidden_states
|
||||
)
|
||||
router_logits = router_logits.to(hidden_states.dtype)
|
||||
router_logits, _ = self.gate(hidden_states)
|
||||
final_hidden = self.experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
@@ -643,9 +642,21 @@ class SarvamMLAForCausalLM(
|
||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _remap_config(config) -> None:
|
||||
"""Default the routing keys the released checkpoints omit."""
|
||||
defaults = {
|
||||
"n_group": 1,
|
||||
"topk_group": 1,
|
||||
}
|
||||
for attr, default in defaults.items():
|
||||
if getattr(config, attr, None) is None:
|
||||
setattr(config, attr, default)
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
self._remap_config(config)
|
||||
quant_config = vllm_config.quant_config
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
|
||||
Reference in New Issue
Block a user