Fix MLA padding and grouped topk routing in the Transformers modelling backend (#49982)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-07-27 18:32:32 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent bf2b45b5d6
commit 99de48e98f
5 changed files with 227 additions and 39 deletions
@@ -6,14 +6,14 @@ import ast
import inspect
import textwrap
import types
from collections.abc import Iterator
from collections.abc import Iterable, Iterator
from dataclasses import dataclass
from itertools import chain
import torch
from torch import fx, nn
from vllm.distributed import tensor_model_parallel_all_gather
from vllm.model_executor.layers.fused_moe import GateLinear
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.models.transformers.fx_utils import (
find_node,
@@ -21,14 +21,10 @@ from vllm.model_executor.models.transformers.fx_utils import (
peel,
trace,
)
from vllm.model_executor.models.transformers.utils import named_state
from vllm.model_executor.models.utils import maybe_prefix, sequence_parallel_chunk
def named_state(module: nn.Module) -> Iterator[tuple[str, torch.Tensor]]:
"""`module`'s own state (i.e. named parameters and buffers)."""
return chain(module.named_parameters(), module.named_buffers())
def _own_returns(node: ast.AST) -> Iterator[ast.Return]:
"""`return` statements in `node`'s own scope, not in nested functions."""
stack = list(ast.iter_child_nodes(node))
@@ -79,6 +75,25 @@ def _is_scalar_gate(module: nn.Module) -> bool:
)
def _forced_dtype(nodes: Iterable[fx.Node]) -> torch.dtype | None:
"""The floating dtype `nodes` cast to, if any.
Computations that must run in higher precision say so in their forward (e.g.
`hidden_states.type(torch.float32)`), so the dtype is readable from the graph
even when the config does not name it."""
for node in nodes:
if node.op not in ("call_method", "call_function"):
continue
name = str(node.target).rsplit(".", 1)[-1]
if name == "float":
return torch.float32
if name in ("to", "type"):
for arg in (*node.args[1:], *node.kwargs.values()):
if isinstance(arg, torch.dtype) and arg.is_floating_point:
return arg
return None
def _reaches(node: fx.Node, key: str) -> set[fx.Node]:
"""Returns the set of nodes reachable from `node` by following `key` edges."""
seen: set[fx.Node] = set()
@@ -131,18 +146,24 @@ class MoEBlockFuser:
scoring_func: str
shared_name: str | None
shared_gate_name: str | None
router_dtype: torch.dtype | None = None
@staticmethod
def _match_router(gate: nn.Module) -> str | None:
"""Matches `topk(score(linear(x)))`, `score` being `softmax`/`sigmoid`."""
if [name for name, _ in named_state(gate)] != ["weight"]:
def _match_router(gate: nn.Module) -> tuple[str, torch.dtype | None] | None:
"""Matches `topk(score(linear(x)))`, `score` being `softmax`/`sigmoid`.
Returns the scoring function and the dtype the router computes in."""
state = {name for name, _ in named_state(gate)}
if "weight" not in state or state - {"weight", "e_score_correction_bias"}:
return None
graph = trace(gate)
if graph is None:
return None
topk = find_node(graph, lambda n: is_op(n, "topk"))
if topk is None:
# The routing top-k is the last one; any earlier one scores expert groups.
topks = [node for node in graph.nodes if is_op(node, "topk")]
if not topks:
return None
topk = topks[-1]
# Exactly one scoring op upstream of the top-k, fed (transitively) by a linear.
scorers = [
n
@@ -152,9 +173,11 @@ class MoEBlockFuser:
if len(scorers) != 1:
return None
scorer = scorers[0]
if not any(is_op(n, "linear") for n in _reaches(scorer, "all_input_nodes")):
logits_cone = _reaches(scorer, "all_input_nodes")
if not any(is_op(n, "linear") for n in logits_cone):
return None
return "softmax" if is_op(scorer, "softmax") else "sigmoid"
scoring_func = "softmax" if is_op(scorer, "softmax") else "sigmoid"
return scoring_func, _forced_dtype(logits_cone)
@staticmethod
def _match_shared_experts(
@@ -198,10 +221,14 @@ class MoEBlockFuser:
if _returns_tuple(type(moe_block)):
return None
# Router: the child that scores + top-k selects.
gate_name = scoring_func = None
gate_name = scoring_func = router_dtype = None
for name, child in moe_block.named_children():
if name != experts_name and (func := cls._match_router(child)) is not None:
gate_name, scoring_func = name, func
if (
name != experts_name
and (router := cls._match_router(child)) is not None
):
gate_name = name
scoring_func, router_dtype = router
break
if gate_name is None or scoring_func is None:
return None
@@ -229,17 +256,23 @@ class MoEBlockFuser:
for name, child in moe_block.named_children():
if name not in accounted and next(named_state(child), None) is not None:
return None
return cls(gate_name, scoring_func, shared_name, shared_gate_name)
return cls(gate_name, scoring_func, shared_name, shared_gate_name, router_dtype)
def gate(self, moe_block: nn.Module, prefix: str) -> ReplicatedLinear:
"""Rebuild the HF gate as a `ReplicatedLinear` for vLLM's fused MoE."""
num_experts, hidden_size = getattr(moe_block, self.gate_name).weight.shape
gate = ReplicatedLinear(
def gate(
self, moe_block: nn.Module, prefix: str, out_dtype: torch.dtype | None = None
) -> GateLinear:
"""Rebuild the HF gate as a `GateLinear` for vLLM's fused MoE."""
hf_gate = getattr(moe_block, self.gate_name)
num_experts, hidden_size = hf_gate.weight.shape
gate = GateLinear(
hidden_size,
num_experts,
bias=False,
out_dtype=out_dtype or self.router_dtype,
prefix=maybe_prefix(prefix, self.gate_name),
)
if (bias := getattr(hf_gate, "e_score_correction_bias", None)) is not None:
gate.register_buffer("e_score_correction_bias", bias)
setattr(moe_block, self.gate_name, gate)
return gate