[Model] Kolibri 1 via the Transformers modeling backend (#60827)
Signed-off-by: Luca Motz <luca.motz@icloud.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Codex
Harry Mellor
parent
e90158d912
commit
698e79f3bd
@@ -1,8 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MoE fuser: route an HF MoE block through `FusedMoE` with vLLM's own routing."""
|
||||
"""MoE fuser: run an HF MoE block's gate, experts and shared experts in `FusedMoE`."""
|
||||
|
||||
import ast
|
||||
import functools
|
||||
import inspect
|
||||
import textwrap
|
||||
import types
|
||||
@@ -11,20 +12,25 @@ from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
from torch import fx, nn
|
||||
from typing_extensions import TypeIs
|
||||
|
||||
from vllm.distributed import tensor_model_parallel_all_gather
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import GateLinear
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
find_node,
|
||||
is_complete,
|
||||
is_op,
|
||||
output_value,
|
||||
peel,
|
||||
trace,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.utils import named_state
|
||||
from vllm.model_executor.models.utils import maybe_prefix, sequence_parallel_chunk
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _own_returns(node: ast.AST) -> Iterator[ast.Return]:
|
||||
"""`return` statements in `node`'s own scope, not in nested functions."""
|
||||
@@ -126,6 +132,35 @@ def _renormalizes(topk: fx.Node) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _unwrap(node: object, moe_block: nn.Module) -> object:
|
||||
"""Strip reshapes, dtype casts and `nn.Identity` calls."""
|
||||
while True:
|
||||
node = peel(node)
|
||||
if (
|
||||
is_op(node, "reshape")
|
||||
or is_op(node, "view")
|
||||
or (
|
||||
isinstance(node, fx.Node)
|
||||
and node.op == "call_module"
|
||||
and isinstance(moe_block.get_submodule(node.target), nn.Identity)
|
||||
)
|
||||
):
|
||||
node = node.args[0]
|
||||
else:
|
||||
return node
|
||||
|
||||
|
||||
def _reads_input(node: object, x: fx.Node, moe_block: nn.Module) -> TypeIs[fx.Node]:
|
||||
"""Is `node` a child module of `moe_block` called on just `x`?"""
|
||||
return (
|
||||
isinstance(node, fx.Node)
|
||||
and node.op == "call_module"
|
||||
and len(node.args) == 1
|
||||
and not node.kwargs
|
||||
and _unwrap(node.args[0], moe_block) is x
|
||||
)
|
||||
|
||||
|
||||
class SharedExpertMLP(nn.Module):
|
||||
"""Wraps an HF shared expert, applying the output gating it is paired with."""
|
||||
|
||||
@@ -162,7 +197,8 @@ class MoEBlockFuser:
|
||||
"""Fuser for MoE block `experts`, `gate` and `shared_experts` (optional)."""
|
||||
|
||||
gate_name: str
|
||||
scoring_func: str
|
||||
scoring_func: str | None
|
||||
"""`None` if routing stays in the Transformers gate."""
|
||||
shared_name: str | None
|
||||
shared_gate_name: str | None
|
||||
router_dtype: torch.dtype | None = None
|
||||
@@ -215,83 +251,142 @@ class MoEBlockFuser:
|
||||
scoring_func = "softmax" if is_softmax else "sigmoid"
|
||||
return scoring_func, _forced_dtype(logits_cone), renormalize
|
||||
|
||||
@staticmethod
|
||||
def _match_experts(
|
||||
graph: fx.Graph, experts_name: str, moe_block: nn.Module
|
||||
) -> tuple[fx.Node, fx.Node, fx.Node] | None:
|
||||
"""Matches `experts(x, ids, weights)` with `_, weights, ids = gate(x)`, `x`
|
||||
being the block's input.
|
||||
|
||||
Returns the experts call, `x` and the gate call."""
|
||||
experts = find_node(
|
||||
graph, lambda n: n.op == "call_module" and n.target == experts_name
|
||||
)
|
||||
if experts is None:
|
||||
return None
|
||||
# Bind by signature so keyword calls match too. Fail closed on any other
|
||||
# argument: the rewritten forward would drop it.
|
||||
forward = inspect.signature(moe_block.get_submodule(experts_name).forward)
|
||||
try:
|
||||
bound = forward.bind(*experts.args, **experts.kwargs)
|
||||
except TypeError:
|
||||
return None
|
||||
if len(bound.arguments) != 3:
|
||||
return None
|
||||
x, ids, weights = (_unwrap(a, moe_block) for a in bound.arguments.values())
|
||||
if not (isinstance(x, fx.Node) and x.op == "placeholder"):
|
||||
return None
|
||||
if not (is_op(ids, "getitem") and is_op(weights, "getitem")):
|
||||
return None
|
||||
gate = ids.args[0]
|
||||
if not (
|
||||
_reads_input(gate, x, moe_block)
|
||||
and ids.args == (gate, 2)
|
||||
and weights.args == (gate, 1)
|
||||
):
|
||||
return None
|
||||
return experts, x, gate
|
||||
|
||||
@staticmethod
|
||||
def _match_shared_experts(
|
||||
graph: fx.Graph, experts: str
|
||||
) -> tuple[str | None, str | None]:
|
||||
"""Detects the shared expert and its optional gate by dataflow."""
|
||||
experts_predicate = lambda n: n.op == "call_module" and n.target == experts
|
||||
if (experts_node := find_node(graph, experts_predicate)) is None:
|
||||
return None, None
|
||||
from_experts = _reaches(experts_node, "users")
|
||||
for add in graph.nodes:
|
||||
if not is_op(add, "add"):
|
||||
node: object, x: fx.Node, moe_block: nn.Module
|
||||
) -> tuple[str, str | None] | None:
|
||||
"""Matches `shared(x)` or `sigmoid(shared_gate(x)) * shared(x)`.
|
||||
|
||||
Returns the shared expert's name and its optional gate's name."""
|
||||
if _reads_input(node, x, moe_block):
|
||||
return node.target, None
|
||||
if not (is_op(node, "mul") and len(node.args) == 2):
|
||||
return None
|
||||
for gated, shared in (node.args, node.args[::-1]):
|
||||
gated, shared = peel(gated), peel(shared)
|
||||
if not (is_op(gated, "sigmoid") and _reads_input(shared, x, moe_block)):
|
||||
continue
|
||||
operands = [a for a in add.args if isinstance(a, fx.Node)]
|
||||
# Exactly one side is the experts' output; the other is the shared path.
|
||||
sides = [a in from_experts for a in operands]
|
||||
if len(operands) != 2 or sides.count(True) != 1:
|
||||
continue
|
||||
cone = _reaches(operands[sides.index(False)], "all_input_nodes")
|
||||
modules = [n for n in cone if n.op == "call_module" and n.target != experts]
|
||||
# A sigmoid wrapping one of those modules marks the shared-expert gate.
|
||||
gate = next(
|
||||
(
|
||||
src
|
||||
for n in cone
|
||||
if is_op(n, "sigmoid")
|
||||
and isinstance(src := peel(n.args[0]), fx.Node)
|
||||
and src in modules
|
||||
),
|
||||
None,
|
||||
)
|
||||
shared = [n for n in modules if n is not gate]
|
||||
if len(shared) != 1:
|
||||
return None, None
|
||||
return shared[0].target, (gate.target if gate is not None else None)
|
||||
return None, None
|
||||
gate = peel(gated.args[0])
|
||||
if _reads_input(gate, x, moe_block) and _is_scalar_gate(
|
||||
moe_block.get_submodule(gate.target)
|
||||
):
|
||||
return shared.target, gate.target
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def match(cls, moe_block: nn.Module, experts_name: str) -> "MoEBlockFuser | None":
|
||||
def match(
|
||||
cls,
|
||||
moe_block: nn.Module,
|
||||
experts_name: str,
|
||||
divides_shared_output: bool = False,
|
||||
) -> "MoEBlockFuser | None":
|
||||
"""Matches a block returning `experts(x, ids, weights) [* c] [+ shared(x)]`
|
||||
with `_, weights, ids = gate(x)`, `x` being the block's input, up to
|
||||
reshapes, casts and `nn.Identity`.
|
||||
|
||||
Args:
|
||||
moe_block: The MoE block.
|
||||
experts_name: The name of the experts in `moe_block`.
|
||||
divides_shared_output: Whether vLLM routing would divide the shared
|
||||
expert's output by `routed_scaling_factor` (fp16 overflow
|
||||
protection), which Transformers decoder layers do not undo.
|
||||
|
||||
Returns:
|
||||
A fuser if more than the experts can run in vLLM, else `None`.
|
||||
|
||||
"""
|
||||
# Standard MoE block returns a single tensor.
|
||||
if _returns_tuple(type(moe_block)):
|
||||
return None
|
||||
# Router: the child that scores + top-k selects.
|
||||
gate_name = scoring_func = router_dtype = renormalize = None
|
||||
for name, child in moe_block.named_children():
|
||||
if (
|
||||
name != experts_name
|
||||
and (router := cls._match_router(child)) is not None
|
||||
):
|
||||
gate_name = name
|
||||
scoring_func, router_dtype, renormalize = router
|
||||
break
|
||||
if gate_name is None or scoring_func is None:
|
||||
if (graph := trace(moe_block)) is None:
|
||||
return None
|
||||
# Shared expert: a child the block adds to the experts' output.
|
||||
shared_name = shared_gate_name = None
|
||||
others = [
|
||||
n
|
||||
for n, _ in moe_block.named_children()
|
||||
if n not in {experts_name, gate_name}
|
||||
]
|
||||
if others:
|
||||
graph = trace(moe_block)
|
||||
if graph is None:
|
||||
return None
|
||||
shared_name, shared_gate_name = cls._match_shared_experts(
|
||||
graph, experts_name
|
||||
if (matched := cls._match_experts(graph, experts_name, moe_block)) is None:
|
||||
return None
|
||||
experts, x, gate = matched
|
||||
unwrap = functools.partial(_unwrap, moe_block=moe_block)
|
||||
gate_name = gate.target
|
||||
router = cls._match_router(getattr(moe_block, gate_name))
|
||||
scoring_func, router_dtype, renormalize = router or (None, None, None)
|
||||
|
||||
def is_routed(n: object) -> bool:
|
||||
return n is experts or (
|
||||
is_op(n, "mul")
|
||||
and not isinstance(n.args[1], fx.Node)
|
||||
and unwrap(n.args[0]) is experts
|
||||
)
|
||||
if shared_gate_name is not None and not _is_scalar_gate(
|
||||
getattr(moe_block, shared_gate_name)
|
||||
):
|
||||
|
||||
# Output: the experts' output, plus the shared expert's if any.
|
||||
routed = out = unwrap(output_value(graph))
|
||||
shared = None
|
||||
if not is_routed(out):
|
||||
if not (is_op(out, "add") and len(out.args) == 2):
|
||||
return None
|
||||
routed, shared = map(unwrap, out.args)
|
||||
if is_routed(shared):
|
||||
routed, shared = shared, routed
|
||||
if not is_routed(routed):
|
||||
return None
|
||||
shared_name = shared_gate_name = None
|
||||
if shared is not None:
|
||||
if (match := cls._match_shared_experts(shared, x, moe_block)) is None:
|
||||
return None
|
||||
shared_name, shared_gate_name = match
|
||||
# Fail closed: `rewrite_forward` runs only the experts and the detected
|
||||
# shared expert, so any other stateful child would be dropped.
|
||||
accounted = {experts_name, gate_name, shared_name, shared_gate_name}
|
||||
for name, child in moe_block.named_children():
|
||||
if name not in accounted and next(named_state(child), None) is not None:
|
||||
return None
|
||||
if scoring_func is not None and shared_name and divides_shared_output:
|
||||
logger.warning_once(
|
||||
"%s could be fused but routing it in vLLM would apply "
|
||||
"`routed_scaling_factor` by dividing the shared expert output, "
|
||||
"which only fp16 overflow protection expects the decoder layer to "
|
||||
"compensate for. Falling back to routing in Transformers; run in "
|
||||
"bfloat16 to fuse it.",
|
||||
type(moe_block).__name__,
|
||||
)
|
||||
scoring_func = router_dtype = renormalize = None
|
||||
# Without vLLM routing, only a shared expert can move to vLLM, and only
|
||||
# if the block does not scale the experts' output (vLLM routing does).
|
||||
if scoring_func is None and (shared_name is None or routed is not experts):
|
||||
return None
|
||||
return cls(
|
||||
gate_name,
|
||||
scoring_func,
|
||||
|
||||
Reference in New Issue
Block a user