Fix MLA padding and grouped topk routing in the Transformers modelling backend (#49982)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
bf2b45b5d6
commit
99de48e98f
@@ -6,14 +6,14 @@ import ast
|
||||
import inspect
|
||||
import textwrap
|
||||
import types
|
||||
from collections.abc import Iterator
|
||||
from collections.abc import Iterable, Iterator
|
||||
from dataclasses import dataclass
|
||||
from itertools import chain
|
||||
|
||||
import torch
|
||||
from torch import fx, nn
|
||||
|
||||
from vllm.distributed import tensor_model_parallel_all_gather
|
||||
from vllm.model_executor.layers.fused_moe import GateLinear
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
find_node,
|
||||
@@ -21,14 +21,10 @@ from vllm.model_executor.models.transformers.fx_utils import (
|
||||
peel,
|
||||
trace,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.utils import named_state
|
||||
from vllm.model_executor.models.utils import maybe_prefix, sequence_parallel_chunk
|
||||
|
||||
|
||||
def named_state(module: nn.Module) -> Iterator[tuple[str, torch.Tensor]]:
|
||||
"""`module`'s own state (i.e. named parameters and buffers)."""
|
||||
return chain(module.named_parameters(), module.named_buffers())
|
||||
|
||||
|
||||
def _own_returns(node: ast.AST) -> Iterator[ast.Return]:
|
||||
"""`return` statements in `node`'s own scope, not in nested functions."""
|
||||
stack = list(ast.iter_child_nodes(node))
|
||||
@@ -79,6 +75,25 @@ def _is_scalar_gate(module: nn.Module) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _forced_dtype(nodes: Iterable[fx.Node]) -> torch.dtype | None:
|
||||
"""The floating dtype `nodes` cast to, if any.
|
||||
|
||||
Computations that must run in higher precision say so in their forward (e.g.
|
||||
`hidden_states.type(torch.float32)`), so the dtype is readable from the graph
|
||||
even when the config does not name it."""
|
||||
for node in nodes:
|
||||
if node.op not in ("call_method", "call_function"):
|
||||
continue
|
||||
name = str(node.target).rsplit(".", 1)[-1]
|
||||
if name == "float":
|
||||
return torch.float32
|
||||
if name in ("to", "type"):
|
||||
for arg in (*node.args[1:], *node.kwargs.values()):
|
||||
if isinstance(arg, torch.dtype) and arg.is_floating_point:
|
||||
return arg
|
||||
return None
|
||||
|
||||
|
||||
def _reaches(node: fx.Node, key: str) -> set[fx.Node]:
|
||||
"""Returns the set of nodes reachable from `node` by following `key` edges."""
|
||||
seen: set[fx.Node] = set()
|
||||
@@ -131,18 +146,24 @@ class MoEBlockFuser:
|
||||
scoring_func: str
|
||||
shared_name: str | None
|
||||
shared_gate_name: str | None
|
||||
router_dtype: torch.dtype | None = None
|
||||
|
||||
@staticmethod
|
||||
def _match_router(gate: nn.Module) -> str | None:
|
||||
"""Matches `topk(score(linear(x)))`, `score` being `softmax`/`sigmoid`."""
|
||||
if [name for name, _ in named_state(gate)] != ["weight"]:
|
||||
def _match_router(gate: nn.Module) -> tuple[str, torch.dtype | None] | None:
|
||||
"""Matches `topk(score(linear(x)))`, `score` being `softmax`/`sigmoid`.
|
||||
|
||||
Returns the scoring function and the dtype the router computes in."""
|
||||
state = {name for name, _ in named_state(gate)}
|
||||
if "weight" not in state or state - {"weight", "e_score_correction_bias"}:
|
||||
return None
|
||||
graph = trace(gate)
|
||||
if graph is None:
|
||||
return None
|
||||
topk = find_node(graph, lambda n: is_op(n, "topk"))
|
||||
if topk is None:
|
||||
# The routing top-k is the last one; any earlier one scores expert groups.
|
||||
topks = [node for node in graph.nodes if is_op(node, "topk")]
|
||||
if not topks:
|
||||
return None
|
||||
topk = topks[-1]
|
||||
# Exactly one scoring op upstream of the top-k, fed (transitively) by a linear.
|
||||
scorers = [
|
||||
n
|
||||
@@ -152,9 +173,11 @@ class MoEBlockFuser:
|
||||
if len(scorers) != 1:
|
||||
return None
|
||||
scorer = scorers[0]
|
||||
if not any(is_op(n, "linear") for n in _reaches(scorer, "all_input_nodes")):
|
||||
logits_cone = _reaches(scorer, "all_input_nodes")
|
||||
if not any(is_op(n, "linear") for n in logits_cone):
|
||||
return None
|
||||
return "softmax" if is_op(scorer, "softmax") else "sigmoid"
|
||||
scoring_func = "softmax" if is_op(scorer, "softmax") else "sigmoid"
|
||||
return scoring_func, _forced_dtype(logits_cone)
|
||||
|
||||
@staticmethod
|
||||
def _match_shared_experts(
|
||||
@@ -198,10 +221,14 @@ class MoEBlockFuser:
|
||||
if _returns_tuple(type(moe_block)):
|
||||
return None
|
||||
# Router: the child that scores + top-k selects.
|
||||
gate_name = scoring_func = None
|
||||
gate_name = scoring_func = router_dtype = None
|
||||
for name, child in moe_block.named_children():
|
||||
if name != experts_name and (func := cls._match_router(child)) is not None:
|
||||
gate_name, scoring_func = name, func
|
||||
if (
|
||||
name != experts_name
|
||||
and (router := cls._match_router(child)) is not None
|
||||
):
|
||||
gate_name = name
|
||||
scoring_func, router_dtype = router
|
||||
break
|
||||
if gate_name is None or scoring_func is None:
|
||||
return None
|
||||
@@ -229,17 +256,23 @@ class MoEBlockFuser:
|
||||
for name, child in moe_block.named_children():
|
||||
if name not in accounted and next(named_state(child), None) is not None:
|
||||
return None
|
||||
return cls(gate_name, scoring_func, shared_name, shared_gate_name)
|
||||
return cls(gate_name, scoring_func, shared_name, shared_gate_name, router_dtype)
|
||||
|
||||
def gate(self, moe_block: nn.Module, prefix: str) -> ReplicatedLinear:
|
||||
"""Rebuild the HF gate as a `ReplicatedLinear` for vLLM's fused MoE."""
|
||||
num_experts, hidden_size = getattr(moe_block, self.gate_name).weight.shape
|
||||
gate = ReplicatedLinear(
|
||||
def gate(
|
||||
self, moe_block: nn.Module, prefix: str, out_dtype: torch.dtype | None = None
|
||||
) -> GateLinear:
|
||||
"""Rebuild the HF gate as a `GateLinear` for vLLM's fused MoE."""
|
||||
hf_gate = getattr(moe_block, self.gate_name)
|
||||
num_experts, hidden_size = hf_gate.weight.shape
|
||||
gate = GateLinear(
|
||||
hidden_size,
|
||||
num_experts,
|
||||
bias=False,
|
||||
out_dtype=out_dtype or self.router_dtype,
|
||||
prefix=maybe_prefix(prefix, self.gate_name),
|
||||
)
|
||||
if (bias := getattr(hf_gate, "e_score_correction_bias", None)) is not None:
|
||||
gate.register_buffer("e_score_correction_bias", bias)
|
||||
setattr(moe_block, self.gate_name, gate)
|
||||
return gate
|
||||
|
||||
|
||||
Reference in New Issue
Block a user