[Model] Kolibri 1 via the Transformers modeling backend (#60827)
Signed-off-by: Luca Motz <luca.motz@icloud.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Codex
Harry Mellor
parent
e90158d912
commit
698e79f3bd
@@ -717,6 +717,7 @@ _TRANSFORMERS_SUPPORTED_MODELS = {
|
||||
"HunYuanDenseV1ForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"HunYuanMoEV1ForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
|
||||
"Jais2ForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"Kolibri1ForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
|
||||
"MellumForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
|
||||
"NanbeigeForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"OlmoForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MoE fuser: route an HF MoE block through `FusedMoE` with vLLM's own routing."""
|
||||
"""MoE fuser: run an HF MoE block's gate, experts and shared experts in `FusedMoE`."""
|
||||
|
||||
import ast
|
||||
import functools
|
||||
import inspect
|
||||
import textwrap
|
||||
import types
|
||||
@@ -11,20 +12,25 @@ from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
from torch import fx, nn
|
||||
from typing_extensions import TypeIs
|
||||
|
||||
from vllm.distributed import tensor_model_parallel_all_gather
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import GateLinear
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
find_node,
|
||||
is_complete,
|
||||
is_op,
|
||||
output_value,
|
||||
peel,
|
||||
trace,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.utils import named_state
|
||||
from vllm.model_executor.models.utils import maybe_prefix, sequence_parallel_chunk
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _own_returns(node: ast.AST) -> Iterator[ast.Return]:
|
||||
"""`return` statements in `node`'s own scope, not in nested functions."""
|
||||
@@ -126,6 +132,35 @@ def _renormalizes(topk: fx.Node) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _unwrap(node: object, moe_block: nn.Module) -> object:
|
||||
"""Strip reshapes, dtype casts and `nn.Identity` calls."""
|
||||
while True:
|
||||
node = peel(node)
|
||||
if (
|
||||
is_op(node, "reshape")
|
||||
or is_op(node, "view")
|
||||
or (
|
||||
isinstance(node, fx.Node)
|
||||
and node.op == "call_module"
|
||||
and isinstance(moe_block.get_submodule(node.target), nn.Identity)
|
||||
)
|
||||
):
|
||||
node = node.args[0]
|
||||
else:
|
||||
return node
|
||||
|
||||
|
||||
def _reads_input(node: object, x: fx.Node, moe_block: nn.Module) -> TypeIs[fx.Node]:
|
||||
"""Is `node` a child module of `moe_block` called on just `x`?"""
|
||||
return (
|
||||
isinstance(node, fx.Node)
|
||||
and node.op == "call_module"
|
||||
and len(node.args) == 1
|
||||
and not node.kwargs
|
||||
and _unwrap(node.args[0], moe_block) is x
|
||||
)
|
||||
|
||||
|
||||
class SharedExpertMLP(nn.Module):
|
||||
"""Wraps an HF shared expert, applying the output gating it is paired with."""
|
||||
|
||||
@@ -162,7 +197,8 @@ class MoEBlockFuser:
|
||||
"""Fuser for MoE block `experts`, `gate` and `shared_experts` (optional)."""
|
||||
|
||||
gate_name: str
|
||||
scoring_func: str
|
||||
scoring_func: str | None
|
||||
"""`None` if routing stays in the Transformers gate."""
|
||||
shared_name: str | None
|
||||
shared_gate_name: str | None
|
||||
router_dtype: torch.dtype | None = None
|
||||
@@ -215,83 +251,142 @@ class MoEBlockFuser:
|
||||
scoring_func = "softmax" if is_softmax else "sigmoid"
|
||||
return scoring_func, _forced_dtype(logits_cone), renormalize
|
||||
|
||||
@staticmethod
|
||||
def _match_experts(
|
||||
graph: fx.Graph, experts_name: str, moe_block: nn.Module
|
||||
) -> tuple[fx.Node, fx.Node, fx.Node] | None:
|
||||
"""Matches `experts(x, ids, weights)` with `_, weights, ids = gate(x)`, `x`
|
||||
being the block's input.
|
||||
|
||||
Returns the experts call, `x` and the gate call."""
|
||||
experts = find_node(
|
||||
graph, lambda n: n.op == "call_module" and n.target == experts_name
|
||||
)
|
||||
if experts is None:
|
||||
return None
|
||||
# Bind by signature so keyword calls match too. Fail closed on any other
|
||||
# argument: the rewritten forward would drop it.
|
||||
forward = inspect.signature(moe_block.get_submodule(experts_name).forward)
|
||||
try:
|
||||
bound = forward.bind(*experts.args, **experts.kwargs)
|
||||
except TypeError:
|
||||
return None
|
||||
if len(bound.arguments) != 3:
|
||||
return None
|
||||
x, ids, weights = (_unwrap(a, moe_block) for a in bound.arguments.values())
|
||||
if not (isinstance(x, fx.Node) and x.op == "placeholder"):
|
||||
return None
|
||||
if not (is_op(ids, "getitem") and is_op(weights, "getitem")):
|
||||
return None
|
||||
gate = ids.args[0]
|
||||
if not (
|
||||
_reads_input(gate, x, moe_block)
|
||||
and ids.args == (gate, 2)
|
||||
and weights.args == (gate, 1)
|
||||
):
|
||||
return None
|
||||
return experts, x, gate
|
||||
|
||||
@staticmethod
|
||||
def _match_shared_experts(
|
||||
graph: fx.Graph, experts: str
|
||||
) -> tuple[str | None, str | None]:
|
||||
"""Detects the shared expert and its optional gate by dataflow."""
|
||||
experts_predicate = lambda n: n.op == "call_module" and n.target == experts
|
||||
if (experts_node := find_node(graph, experts_predicate)) is None:
|
||||
return None, None
|
||||
from_experts = _reaches(experts_node, "users")
|
||||
for add in graph.nodes:
|
||||
if not is_op(add, "add"):
|
||||
node: object, x: fx.Node, moe_block: nn.Module
|
||||
) -> tuple[str, str | None] | None:
|
||||
"""Matches `shared(x)` or `sigmoid(shared_gate(x)) * shared(x)`.
|
||||
|
||||
Returns the shared expert's name and its optional gate's name."""
|
||||
if _reads_input(node, x, moe_block):
|
||||
return node.target, None
|
||||
if not (is_op(node, "mul") and len(node.args) == 2):
|
||||
return None
|
||||
for gated, shared in (node.args, node.args[::-1]):
|
||||
gated, shared = peel(gated), peel(shared)
|
||||
if not (is_op(gated, "sigmoid") and _reads_input(shared, x, moe_block)):
|
||||
continue
|
||||
operands = [a for a in add.args if isinstance(a, fx.Node)]
|
||||
# Exactly one side is the experts' output; the other is the shared path.
|
||||
sides = [a in from_experts for a in operands]
|
||||
if len(operands) != 2 or sides.count(True) != 1:
|
||||
continue
|
||||
cone = _reaches(operands[sides.index(False)], "all_input_nodes")
|
||||
modules = [n for n in cone if n.op == "call_module" and n.target != experts]
|
||||
# A sigmoid wrapping one of those modules marks the shared-expert gate.
|
||||
gate = next(
|
||||
(
|
||||
src
|
||||
for n in cone
|
||||
if is_op(n, "sigmoid")
|
||||
and isinstance(src := peel(n.args[0]), fx.Node)
|
||||
and src in modules
|
||||
),
|
||||
None,
|
||||
)
|
||||
shared = [n for n in modules if n is not gate]
|
||||
if len(shared) != 1:
|
||||
return None, None
|
||||
return shared[0].target, (gate.target if gate is not None else None)
|
||||
return None, None
|
||||
gate = peel(gated.args[0])
|
||||
if _reads_input(gate, x, moe_block) and _is_scalar_gate(
|
||||
moe_block.get_submodule(gate.target)
|
||||
):
|
||||
return shared.target, gate.target
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def match(cls, moe_block: nn.Module, experts_name: str) -> "MoEBlockFuser | None":
|
||||
def match(
|
||||
cls,
|
||||
moe_block: nn.Module,
|
||||
experts_name: str,
|
||||
divides_shared_output: bool = False,
|
||||
) -> "MoEBlockFuser | None":
|
||||
"""Matches a block returning `experts(x, ids, weights) [* c] [+ shared(x)]`
|
||||
with `_, weights, ids = gate(x)`, `x` being the block's input, up to
|
||||
reshapes, casts and `nn.Identity`.
|
||||
|
||||
Args:
|
||||
moe_block: The MoE block.
|
||||
experts_name: The name of the experts in `moe_block`.
|
||||
divides_shared_output: Whether vLLM routing would divide the shared
|
||||
expert's output by `routed_scaling_factor` (fp16 overflow
|
||||
protection), which Transformers decoder layers do not undo.
|
||||
|
||||
Returns:
|
||||
A fuser if more than the experts can run in vLLM, else `None`.
|
||||
|
||||
"""
|
||||
# Standard MoE block returns a single tensor.
|
||||
if _returns_tuple(type(moe_block)):
|
||||
return None
|
||||
# Router: the child that scores + top-k selects.
|
||||
gate_name = scoring_func = router_dtype = renormalize = None
|
||||
for name, child in moe_block.named_children():
|
||||
if (
|
||||
name != experts_name
|
||||
and (router := cls._match_router(child)) is not None
|
||||
):
|
||||
gate_name = name
|
||||
scoring_func, router_dtype, renormalize = router
|
||||
break
|
||||
if gate_name is None or scoring_func is None:
|
||||
if (graph := trace(moe_block)) is None:
|
||||
return None
|
||||
# Shared expert: a child the block adds to the experts' output.
|
||||
shared_name = shared_gate_name = None
|
||||
others = [
|
||||
n
|
||||
for n, _ in moe_block.named_children()
|
||||
if n not in {experts_name, gate_name}
|
||||
]
|
||||
if others:
|
||||
graph = trace(moe_block)
|
||||
if graph is None:
|
||||
return None
|
||||
shared_name, shared_gate_name = cls._match_shared_experts(
|
||||
graph, experts_name
|
||||
if (matched := cls._match_experts(graph, experts_name, moe_block)) is None:
|
||||
return None
|
||||
experts, x, gate = matched
|
||||
unwrap = functools.partial(_unwrap, moe_block=moe_block)
|
||||
gate_name = gate.target
|
||||
router = cls._match_router(getattr(moe_block, gate_name))
|
||||
scoring_func, router_dtype, renormalize = router or (None, None, None)
|
||||
|
||||
def is_routed(n: object) -> bool:
|
||||
return n is experts or (
|
||||
is_op(n, "mul")
|
||||
and not isinstance(n.args[1], fx.Node)
|
||||
and unwrap(n.args[0]) is experts
|
||||
)
|
||||
if shared_gate_name is not None and not _is_scalar_gate(
|
||||
getattr(moe_block, shared_gate_name)
|
||||
):
|
||||
|
||||
# Output: the experts' output, plus the shared expert's if any.
|
||||
routed = out = unwrap(output_value(graph))
|
||||
shared = None
|
||||
if not is_routed(out):
|
||||
if not (is_op(out, "add") and len(out.args) == 2):
|
||||
return None
|
||||
routed, shared = map(unwrap, out.args)
|
||||
if is_routed(shared):
|
||||
routed, shared = shared, routed
|
||||
if not is_routed(routed):
|
||||
return None
|
||||
shared_name = shared_gate_name = None
|
||||
if shared is not None:
|
||||
if (match := cls._match_shared_experts(shared, x, moe_block)) is None:
|
||||
return None
|
||||
shared_name, shared_gate_name = match
|
||||
# Fail closed: `rewrite_forward` runs only the experts and the detected
|
||||
# shared expert, so any other stateful child would be dropped.
|
||||
accounted = {experts_name, gate_name, shared_name, shared_gate_name}
|
||||
for name, child in moe_block.named_children():
|
||||
if name not in accounted and next(named_state(child), None) is not None:
|
||||
return None
|
||||
if scoring_func is not None and shared_name and divides_shared_output:
|
||||
logger.warning_once(
|
||||
"%s could be fused but routing it in vLLM would apply "
|
||||
"`routed_scaling_factor` by dividing the shared expert output, "
|
||||
"which only fp16 overflow protection expects the decoder layer to "
|
||||
"compensate for. Falling back to routing in Transformers; run in "
|
||||
"bfloat16 to fuse it.",
|
||||
type(moe_block).__name__,
|
||||
)
|
||||
scoring_func = router_dtype = renormalize = None
|
||||
# Without vLLM routing, only a shared expert can move to vLLM, and only
|
||||
# if the block does not scale the experts' output (vLLM routing does).
|
||||
if scoring_func is None and (shared_name is None or routed is not experts):
|
||||
return None
|
||||
return cls(
|
||||
gate_name,
|
||||
scoring_func,
|
||||
|
||||
@@ -127,6 +127,43 @@ direct_register_custom_op(
|
||||
)
|
||||
|
||||
|
||||
def _hf_topk_router_precomputed(
|
||||
hidden_states: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
moe_state: TransformersMoEState,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Return `(topk_weights, topk_ids)` the HF block forward already computed:
|
||||
weights arrive as `gating_output`, ids via `moe_state`."""
|
||||
topk_weights = gating_output
|
||||
topk_ids = moe_state.topk_ids
|
||||
assert topk_ids is not None
|
||||
# Handle all gather in expert parallel
|
||||
if topk_ids.size(0) != hidden_states.size(0):
|
||||
dp_metadata = get_forward_context().dp_metadata
|
||||
assert dp_metadata is not None
|
||||
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
|
||||
assert sizes is not None
|
||||
is_sp = moe_state.is_sequence_parallel
|
||||
group = get_ep_group() if is_sp else get_dp_group()
|
||||
assert sizes[group.rank_in_group] == topk_ids.shape[0]
|
||||
(topk_ids,) = group.all_gatherv([topk_ids], 0, sizes)
|
||||
return topk_weights, topk_ids
|
||||
|
||||
|
||||
def _hf_topk_router(
|
||||
hidden_states: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
gate: nn.Module,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Route with a Transformers gate returning `(_, topk_weights, topk_ids)`."""
|
||||
_, topk_weights, topk_ids = gate(hidden_states)
|
||||
return topk_weights, topk_ids
|
||||
|
||||
|
||||
class TransformersRoutedExperts(RoutedExperts):
|
||||
def get_expert_mapping(
|
||||
self,
|
||||
@@ -310,26 +347,17 @@ class MoEMixin(MixtureOfExperts, Base):
|
||||
is_fused_checkpoint_transposed=is_transposed,
|
||||
routed_experts_cls=TransformersRoutedExperts,
|
||||
)
|
||||
fuser = MoEBlockFuser.match(moe_block, experts_name)
|
||||
# _maybe_apply_routed_scale_to_output edge case. Transformers
|
||||
# decoder layers do not compensate for dividing by scaling factor.
|
||||
reaches_fp16_trick = (
|
||||
divides_shared_output = (
|
||||
routed_scaling_factor != 1.0
|
||||
and apply_routed_scale_to_output
|
||||
and self.model_config.dtype == torch.float16
|
||||
and fuser is not None
|
||||
and fuser.shared_name is not None
|
||||
)
|
||||
if reaches_fp16_trick:
|
||||
logger.warning_once(
|
||||
"%s could be fused but routing it in vLLM would apply "
|
||||
"`routed_scaling_factor` by dividing the shared expert "
|
||||
"output, which only fp16 overflow protection expects the "
|
||||
"decoder layer to compensate for. Falling back to routing "
|
||||
"in Transformers; run in bfloat16 to fuse it.",
|
||||
moe_block_cls,
|
||||
)
|
||||
if fuser is not None and not reaches_fp16_trick:
|
||||
fuser = MoEBlockFuser.match(
|
||||
moe_block, experts_name, divides_shared_output
|
||||
)
|
||||
if fuser is not None:
|
||||
# MoE block forward is fully replaced.
|
||||
# gate/router and shared expert (if any) runs in MoERunner.
|
||||
shared_experts = fuser.shared_experts(moe_block, prefix)
|
||||
@@ -341,39 +369,52 @@ class MoEMixin(MixtureOfExperts, Base):
|
||||
down_name = glu_fuser.down_name
|
||||
if down_name is not None:
|
||||
shared_down_projs.append((hf_shared, down_name))
|
||||
# Prefer config, otherwise read it from fuser.
|
||||
router_dtype = config_router_dtype or fuser.router_dtype
|
||||
gate = fuser.gate(moe_block, prefix, router_dtype)
|
||||
if scoring_func := fuser.scoring_func:
|
||||
# Internal routing kwargs.
|
||||
# Prefer config, otherwise read it from fuser.
|
||||
router_dtype = config_router_dtype or fuser.router_dtype
|
||||
gate = fuser.gate(moe_block, prefix, router_dtype)
|
||||
kwargs |= dict(
|
||||
scoring_func=scoring_func,
|
||||
renormalize=(
|
||||
layer_renormalize
|
||||
if fuser.renormalize is None
|
||||
else fuser.renormalize
|
||||
),
|
||||
gate=gate,
|
||||
)
|
||||
if router_dtype is not None:
|
||||
kwargs["router_logits_dtype"] = router_dtype
|
||||
if use_grouped_topk:
|
||||
kwargs |= grouped_topk_routing_kwargs
|
||||
if routed_scaling_factor != 1.0:
|
||||
kwargs |= routed_scaling_factor_kwargs
|
||||
bias = getattr(gate, "e_score_correction_bias", None)
|
||||
if bias is not None:
|
||||
kwargs["e_score_correction_bias"] = bias
|
||||
else:
|
||||
# External routing kwargs
|
||||
kwargs |= dict(
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
custom_routing_function=partial(
|
||||
_hf_topk_router,
|
||||
gate=getattr(moe_block, fuser.gate_name),
|
||||
),
|
||||
)
|
||||
|
||||
kwargs |= dict(
|
||||
scoring_func=fuser.scoring_func,
|
||||
renormalize=(
|
||||
layer_renormalize
|
||||
if fuser.renormalize is None
|
||||
else fuser.renormalize
|
||||
),
|
||||
is_sequence_parallel=(
|
||||
self.parallel_config.use_sequence_parallel_moe
|
||||
),
|
||||
gate=gate,
|
||||
shared_experts=shared_experts,
|
||||
)
|
||||
if router_dtype is not None:
|
||||
kwargs["router_logits_dtype"] = router_dtype
|
||||
if use_grouped_topk:
|
||||
kwargs |= grouped_topk_routing_kwargs
|
||||
if routed_scaling_factor != 1.0:
|
||||
kwargs |= routed_scaling_factor_kwargs
|
||||
bias = getattr(gate, "e_score_correction_bias", None)
|
||||
if bias is not None:
|
||||
kwargs["e_score_correction_bias"] = bias
|
||||
fuser.rewrite_forward(moe_block)
|
||||
routed = "gate + experts"
|
||||
if fuser.shared_name:
|
||||
routed += " + shared experts"
|
||||
logger.info_once(
|
||||
"Fused: %s (%s) -> MoERunner (internal routing)",
|
||||
routed,
|
||||
"Fused: gate + experts%s (%s) -> MoERunner (%s routing)",
|
||||
" + shared experts" if fuser.shared_name else "",
|
||||
moe_block_cls,
|
||||
"external" if fuser.scoring_func is None else "internal",
|
||||
)
|
||||
else:
|
||||
# MoE block forward is unmodified.
|
||||
@@ -381,35 +422,11 @@ class MoEMixin(MixtureOfExperts, Base):
|
||||
# We then smuggle the topk_ids in using a custom op.
|
||||
moe_state = TransformersMoEState()
|
||||
|
||||
def custom_routing_function(
|
||||
hidden_states: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
moe_state: TransformersMoEState,
|
||||
):
|
||||
"""Return `topk_weights` from `gating_output` and the
|
||||
`topk_ids` we stored in the layer earlier."""
|
||||
topk_weights = gating_output
|
||||
topk_ids = moe_state.topk_ids
|
||||
assert topk_ids is not None
|
||||
# Handle all gather in expert parallel
|
||||
if topk_ids.size(0) != hidden_states.size(0):
|
||||
dp_metadata = get_forward_context().dp_metadata
|
||||
assert dp_metadata is not None
|
||||
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
|
||||
assert sizes is not None
|
||||
is_sp = moe_state.is_sequence_parallel
|
||||
group = get_ep_group() if is_sp else get_dp_group()
|
||||
assert sizes[group.rank_in_group] == topk_ids.shape[0]
|
||||
(topk_ids,) = group.all_gatherv([topk_ids], 0, sizes)
|
||||
return topk_weights, topk_ids
|
||||
|
||||
kwargs |= dict(
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
custom_routing_function=partial(
|
||||
custom_routing_function, moe_state=moe_state
|
||||
_hf_topk_router_precomputed, moe_state=moe_state
|
||||
),
|
||||
runner_cls=TransformersMoERunner,
|
||||
runner_args={"moe_state": moe_state},
|
||||
|
||||
Reference in New Issue
Block a user