Add MoNe experts pruning support (#50415)

Signed-off-by: Ilya Boytsov <ieboytsov@nebius.com>
Signed-off-by: Misha Goin <mgoin64@gmail.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Ilya Boytsov <ieboytsov@nebius.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilya Boytsov
2026-10-12 23:18:43 +00:00
committed by GitHub
co-authored by Ilya Boytsov Codex Misha Goin Claude Opus 5.5
parent 1484266938
commit 189bbfb9f4
13 changed files with 1458 additions and 4 deletions
@@ -1293,6 +1293,8 @@ class FusedMoEConfig:
has_bias: bool = False
is_lora_enabled: bool = False
has_hash_routing: bool = False
# Routes are adjusted between routing and expert execution.
require_decomposed_backend: bool = False
shared_expert_prefix: str | None = None
# When True, the MoE skips its final cross-rank all-reduce (and the separate
@@ -0,0 +1,380 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Constant expert substitution for MoNE-pruned checkpoints.
MoNE (https://arxiv.org/abs/2507.00390) replaces redundant experts with
"novices" that return a constant vector. Checkpoints list the replaced experts
per decoder layer either in ``config.approximate_experts``, with constants named
``<experts prefix>.<expert id>.approx_value``, or in
``compression_config.transform_config.expert_substitution``, with explicitly
named constants. Retained experts keep their logical checkpoint names and are
loaded into compact physical rows.
"""
from collections.abc import Mapping
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any
import torch
from torch import nn
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
UnquantizedFusedMoEMethod,
)
if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe.runner.shared_experts import SharedExperts
APPROX_VALUE_SUFFIX = ".approx_value"
def decoder_layer_index(path: str) -> int | None:
"""Return the only integer component of a module path, if unique."""
indices = [int(part) for part in path.split(".") if part.isdigit()]
return indices[0] if len(indices) == 1 else None
@dataclass(frozen=True)
class ExpertSubstitutionSpec:
# Sorted substituted expert IDs per decoder layer.
experts: dict[int, tuple[int, ...]]
# Explicitly named constant tensors and the (layer, expert) pairs they
# supply. Without an entry, constants use the ``approx_value`` naming.
value_names: dict[str, tuple[tuple[int, int], ...]] = field(default_factory=dict)
# The only router semantics that the constant-v1 format supports.
_ROUTER_SEMANTICS = {
"preserve_logical_expert_ids": True,
"preserve_router_weights": True,
"renormalize_after_substitution": False,
}
def _expert_ids(layer_idx: int, expert_ids: Any) -> tuple[int, ...]:
ids = tuple(sorted(int(expert_id) for expert_id in expert_ids or ()))
if len(set(ids)) != len(ids) or any(expert_id < 0 for expert_id in ids):
raise ValueError(
f"substituted experts of layer {layer_idx} must be unique, "
f"non-negative expert IDs, got {list(ids)}"
)
return ids
def _parse_approximate_experts(raw: Any) -> ExpertSubstitutionSpec:
if not isinstance(raw, Mapping):
raise ValueError("approximate_experts must map layer indices to expert IDs")
experts: dict[int, tuple[int, ...]] = {}
for layer, expert_ids in raw.items():
try:
layer_idx = int(layer)
experts[layer_idx] = _expert_ids(layer_idx, expert_ids)
except (TypeError, ValueError) as e:
raise ValueError(
f"approximate_experts has an invalid entry for layer {layer!r}"
) from e
return ExpertSubstitutionSpec({k: v for k, v in experts.items() if v})
def _parse_expert_substitution(raw: Any) -> ExpertSubstitutionSpec:
experts: dict[int, tuple[int, ...]] = {}
value_names: dict[str, list[tuple[int, int]]] = {}
try:
if raw["version"] != 1 or raw["router_semantics"] != _ROUTER_SEMANTICS:
raise ValueError("only version 1 with unchanged routing is supported")
for path, target in raw["targets"].items():
layer_idx = decoder_layer_index(path)
if layer_idx is None or layer_idx in experts:
raise ValueError(f"target {path!r} must name one decoder layer")
if target["weight_layout"] != "compact_retained_experts":
raise ValueError(f"unsupported weight layout for {path!r}")
for expert_id, replacement in target["replacements"].items():
if replacement["format"] != "constant-v1":
raise ValueError(f"unsupported replacement format for {path!r}")
value_names.setdefault(replacement["tensors"]["value"], []).append(
(layer_idx, int(expert_id))
)
experts[layer_idx] = _expert_ids(layer_idx, target["replacements"])
except (AttributeError, KeyError, TypeError, ValueError) as e:
raise ValueError(f"invalid expert_substitution metadata: {e}") from e
return ExpertSubstitutionSpec(
experts, {name: tuple(pairs) for name, pairs in value_names.items()}
)
def get_expert_substitution_spec(model_config: Any) -> ExpertSubstitutionSpec | None:
"""Parse the substituted experts declared in a model's HF config."""
hf_configs = [
getattr(model_config, name, None) for name in ("hf_text_config", "hf_config")
]
approximate = None
for config in hf_configs:
approximate = getattr(config, "approximate_experts", None)
if approximate is not None:
break
compression_config = getattr(hf_configs[1], "compression_config", None)
transform_config = (
compression_config.get("transform_config")
if isinstance(compression_config, Mapping)
else None
)
declared = (
transform_config.get("expert_substitution")
if isinstance(transform_config, Mapping)
else None
)
if approximate is not None and declared is not None:
raise ValueError(
"config declares both approximate_experts and "
"compression_config.transform_config.expert_substitution"
)
if approximate is not None:
spec = _parse_approximate_experts(approximate)
elif declared is not None:
spec = _parse_expert_substitution(declared)
else:
return None
return spec if spec.experts else None
class ConstantExpertSubstitution(nn.Module):
"""Constant outputs for the substituted experts of one MoE layer.
Router outputs keep logical expert IDs. ``transform_routes`` maps retained
experts to compact physical rows and turns substituted routes into
zero-weight routes to physical expert 0, so any decomposed MoE backend can
run them unchanged.
"""
def __init__(
self,
layer_idx: int,
num_logical_experts: int,
substituted_expert_ids: tuple[int, ...],
hidden_size: int,
params_dtype: torch.dtype,
) -> None:
super().__init__()
substituted = set(substituted_expert_ids)
if not substituted <= set(range(num_logical_experts)):
raise ValueError(
f"approximate_experts for layer {layer_idx} must be within "
f"[0, {num_logical_experts}), got {sorted(substituted)}"
)
if len(substituted) == num_logical_experts:
raise ValueError(
f"approximate_experts for layer {layer_idx} must retain at least "
"one expert"
)
self.layer_idx = layer_idx
self.substituted_expert_ids = tuple(sorted(substituted))
retained = [i for i in range(num_logical_experts) if i not in substituted]
self.num_compute_experts = len(retained)
# Retained experts map to their physical row r >= 0, substituted
# experts to -1 - (their row in `values`).
routes = [0] * num_logical_experts
for row, expert_id in enumerate(retained):
routes[expert_id] = row
for row, expert_id in enumerate(self.substituted_expert_ids):
routes[expert_id] = -1 - row
self._routes = tuple(routes)
self.register_buffer(
"expert_substitution_routes",
torch.tensor(routes, dtype=torch.int32),
persistent=False,
)
# NaN marks values that the checkpoint has not provided yet.
self.values = nn.Parameter(
torch.full((len(substituted), hidden_size), torch.nan, dtype=params_dtype),
requires_grad=False,
)
self.values.weight_loader = self.weight_loader
@property
def num_logical_experts(self) -> int:
return len(self._routes)
def physical_expert_id(self, expert_id: int) -> int:
"""Physical row of a retained expert, or -1 for a substituted one."""
return max(self._routes[expert_id], -1)
def weight_loader(
self, param: nn.Parameter, loaded_weight: torch.Tensor, expert_id: int
) -> None:
if not 0 <= expert_id < self.num_logical_experts or (
self._routes[expert_id] >= 0
):
raise ValueError(
f"expert {expert_id} of layer {self.layer_idx} is not listed in "
"approximate_experts"
)
row = param.data[-1 - self._routes[expert_id]]
if loaded_weight.shape != row.shape:
raise ValueError(
f"approx_value for expert {expert_id} of layer {self.layer_idx} has "
f"shape {tuple(loaded_weight.shape)}, expected {tuple(row.shape)}"
)
row.copy_(loaded_weight)
def validate_loaded_values(self, prefix: str) -> None:
missing_rows = torch.isnan(self.values).any(dim=1).nonzero().flatten().tolist()
if missing_rows:
missing = [self.substituted_expert_ids[row] for row in missing_rows]
raise ValueError(
f"{prefix} is missing approx_value for {len(missing)} "
f"expert(s), first missing expert IDs: {missing[:8]}"
)
def transform_routes(
self,
hidden_states: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
) -> torch.Tensor:
"""Remap routes in place and return the constant experts' output."""
valid = (topk_ids >= 0) & (topk_ids < self.num_logical_experts)
routes = self.expert_substitution_routes[
torch.where(valid, topk_ids, torch.zeros_like(topk_ids)).long()
].long()
substituted = valid & (routes < 0)
# Sum each token's router weights per substituted expert in FP32; other
# routes land in a trailing column that is dropped.
num_values, value_size = self.values.shape
scales = topk_weights.new_zeros(
(topk_ids.shape[0], num_values + 1), dtype=torch.float32
)
scales.scatter_add_(
1,
torch.where(substituted, -1 - routes, num_values),
topk_weights.float(),
)
output = torch.zeros_like(hidden_states)
output[..., :value_size] = scales[:, :num_values] @ self.values.float()
topk_weights.masked_fill_(~valid | substituted, 0.0)
topk_ids.copy_(routes.clamp_min(0).to(topk_ids.dtype))
return output
def make_expert_substitution(
model_config: Any,
prefix: str,
num_experts: int,
hidden_size: int,
params_dtype: torch.dtype | None,
) -> ConstantExpertSubstitution | None:
spec = get_expert_substitution_spec(model_config)
if spec is None:
return None
layer_idx = decoder_layer_index(prefix)
if layer_idx is None:
raise ValueError(
f"cannot match MoE layer {prefix!r} to substituted experts: expected "
"exactly one layer index in its prefix"
)
expert_ids = spec.experts.get(layer_idx)
if expert_ids is None:
return None
return ConstantExpertSubstitution(
layer_idx,
num_experts,
expert_ids,
hidden_size,
params_dtype or torch.get_default_dtype(),
)
class SubstitutedRoutedExperts(RoutedExperts):
"""Routed experts that hold weights only for retained experts."""
def __init__(
self, *args: Any, expert_substitution: ConstantExpertSubstitution, **kwargs: Any
):
super().__init__(*args, **kwargs)
self.expert_substitution = expert_substitution
moe_config = self.moe_config
parallel_config = moe_config.moe_parallel_config
unsupported = [
name
for name, enabled in (
(
"quantized MoE weights",
not isinstance(self.quant_method, UnquantizedFusedMoEMethod),
),
("monolithic MoE backends", self.quant_method.is_monolithic),
("MoE LoRA", moe_config.is_lora_enabled),
(
"EPLB",
parallel_config.enable_eplb
or moe_config.num_experts
!= expert_substitution.num_compute_experts,
),
(
"fused shared experts",
self.expert_map_manager.num_fused_shared_experts > 0,
),
("expert parallelism", parallel_config.use_ep),
("data parallelism", parallel_config.dp_size > 1),
("prefill context parallelism", parallel_config.pcp_size > 1),
("sequence parallelism", parallel_config.is_sequence_parallel),
("router weights on expert inputs", self.apply_router_weight_on_input),
)
if enabled
]
if unsupported:
raise NotImplementedError(
"expert substitution does not support: " + ", ".join(unsupported)
)
def _map_global_expert_id_to_local_expert_id(self, expert_id: int) -> int:
# Checkpoints name retained experts by their logical ID.
physical_id = self.expert_substitution.physical_expert_id(expert_id)
if physical_id < 0:
return -1
return super()._map_global_expert_id_to_local_expert_id(physical_id)
def get_expert_mapping(
self,
ckpt_gate_proj_name: str | None = None,
ckpt_down_proj_name: str | None = None,
ckpt_up_proj_name: str | None = None,
include_fused: bool = False,
) -> list[tuple[str, str, int, str]]:
# Fused checkpoint tensors would cover every logical expert at once.
return self.build_expert_params_mapping(
ckpt_gate_proj_name or self.ckpt_gate_proj_name,
ckpt_down_proj_name or self.ckpt_down_proj_name,
ckpt_up_proj_name or self.ckpt_up_proj_name,
num_experts=self.expert_substitution.num_logical_experts,
routed_experts_prefix="",
lora_base_layer_prefix=self.lora_base_layer_prefix,
)
def forward_modular(
self,
x: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
shared_experts: "SharedExperts | None" = None,
shared_experts_input: torch.Tensor | None = None,
) -> torch.Tensor:
constant_output = self.expert_substitution.transform_routes(
x, topk_weights, topk_ids
)
output = super().forward_modular(
x, topk_weights, topk_ids, shared_experts, shared_experts_input
)
if isinstance(output, UnfinalizedMoEOutput):
raise NotImplementedError(
"expert substitution does not support deferred MoE finalize"
)
# The constant output is replicated across TP ranks, whose partial
# outputs are all-reduced later, so add it on one rank only.
if self.moe_config.tp_rank == 0:
output.add_(constant_output[..., : output.shape[-1]])
return output
+25 -1
View File
@@ -23,6 +23,10 @@ from vllm.model_executor.layers.fused_moe.config import (
from vllm.model_executor.layers.fused_moe.expert_map_manager import (
ExpertMapManager,
)
from vllm.model_executor.layers.fused_moe.expert_substitution import (
SubstitutedRoutedExperts,
make_expert_substitution,
)
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
from vllm.model_executor.layers.fused_moe.router.fused_moe_router import (
FusedMoERouter,
@@ -253,6 +257,12 @@ def FusedMoEFactory(
fuse_shared_experts,
)
)
expert_substitution = make_expert_substitution(
vllm_config.model_config, layer_name, num_experts, hidden_size, params_dtype
)
if expert_substitution is not None:
# Route over all logical experts but allocate only the retained ones.
global_num_experts -= len(expert_substitution.substituted_expert_ids)
# Initialize EPLB manager (or None?)
eplb_state: EplbLayerState | None = None
@@ -367,6 +377,7 @@ def FusedMoEFactory(
device=vllm_config.device_config.device,
routing_method=router.routing_method_type, # Not ideal
has_hash_routing=hash_indices_table is not None,
require_decomposed_backend=expert_substitution is not None,
swiglu_limit=swiglu_limit,
swiglu_alpha=swiglu_alpha,
swiglu_beta=swiglu_beta,
@@ -380,7 +391,20 @@ def FusedMoEFactory(
# Create RoutedExperts instance BEFORE create_weights()
# This will hold all expert weight parameters
if routed_experts_cls is None:
if expert_substitution is not None:
if routed_experts_cls not in (None, RoutedExperts) or (
routed_input_transform is not None or routed_output_transform is not None
):
raise NotImplementedError(
"expert substitution requires the default RoutedExperts and no "
"routed input or output transforms"
)
routed_experts_cls = SubstitutedRoutedExperts
routed_experts_args = {
**(routed_experts_args or {}),
"expert_substitution": expert_substitution,
}
elif routed_experts_cls is None:
routed_experts_cls = RoutedExperts
assert params_dtype is not None
@@ -569,6 +569,8 @@ class FusedMoEExperts(ABC):
)
elif moe_config.has_hash_routing and cls.is_monolithic():
return False, _make_reason("hash routing")
elif moe_config.require_decomposed_backend and cls.is_monolithic():
return False, _make_reason("routes adjusted after routing")
elif not cls._supports_routing_method(
moe_config.routing_method, weight_key, activation_key
):
@@ -84,6 +84,8 @@ class TensorizerLoader(BaseModelLoader):
model = initialize_model(vllm_config=vllm_config, prefix=prefix)
model.load_weights(self._get_weights_iterator())
if hasattr(model, "process_weights_after_loading"):
model.process_weights_after_loading()
return model.eval()
def download_model(self, model_config: ModelConfig) -> None:
@@ -84,6 +84,12 @@ def initialize_model(
if model_class is None:
model_class, _ = get_model_architecture(model_config)
from vllm.model_executor.models.expert_substitution import (
as_expert_substitution_model,
)
model_class = as_expert_substitution_model(model_class, model_config)
if vllm_config.quant_config is not None:
configure_quant_config(vllm_config.quant_config, model_class)
@@ -0,0 +1,109 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
from functools import wraps
from typing import Any, TypeVar
import torch
from torch import nn
from vllm.model_executor.layers.fused_moe.expert_substitution import (
APPROX_VALUE_SUFFIX,
ConstantExpertSubstitution,
decoder_layer_index,
get_expert_substitution_spec,
)
from vllm.model_executor.model_loader.reload import finalize_layerwise_reload
_T = TypeVar("_T", bound=nn.Module)
def _collect_expert_substitutions(
model: nn.Module,
) -> dict[int, tuple[str, ConstantExpertSubstitution]]:
substitutions: dict[int, tuple[str, ConstantExpertSubstitution]] = {}
for name, module in model.named_modules():
if not isinstance(module, ConstantExpertSubstitution):
continue
if module.layer_idx in substitutions:
raise ValueError(
f"layer {module.layer_idx} has more than one MoE with substituted "
"experts"
)
substitutions[module.layer_idx] = (f"{name}.values", module)
return substitutions
def _parse_approx_value_name(name: str) -> tuple[int, int] | None:
"""Return (layer index, expert ID) of an ``approx_value`` tensor name."""
if not name.endswith(APPROX_VALUE_SUFFIX):
return None
experts_path, _, expert_id = name.removesuffix(APPROX_VALUE_SUFFIX).rpartition(".")
layer_idx = decoder_layer_index(experts_path)
if layer_idx is None or not expert_id.isdigit():
return None
return layer_idx, int(expert_id)
def as_expert_substitution_model(model_cls: type[_T], model_config: Any) -> type[_T]:
"""Load constant expert values at the ``load_weights`` boundary.
Initial loading and direct weight updates both go through ``load_weights``.
Explicitly named values are always consumed and loaded if their layer is
built by this model. ``approx_value`` tensors of layers built elsewhere
(other pipeline stages, MTP layers) are passed through, so the model's own
loader skips them like any other weight of a layer it does not own.
"""
spec = get_expert_substitution_spec(model_config)
if spec is None:
return model_cls
value_names = spec.value_names
class ExpertSubstitutionModel(model_cls): # type: ignore[valid-type, misc]
@wraps(model_cls.__init__) # type: ignore[misc]
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._expert_substitutions = _collect_expert_substitutions(self)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
if any(s.values.is_meta for _, s in self._expert_substitutions.values()):
# Put the previous weights back so the model stays usable.
finalize_layerwise_reload(self, model_config)
raise NotImplementedError(
"expert substitution does not support layerwise weight reload"
)
loaded_substitutions: set[str] = set()
def remaining_weights():
for name, loaded_weight in weights:
targets = value_names.get(name)
if targets is None:
key = _parse_approx_value_name(name)
if key is None or key[0] not in self._expert_substitutions:
yield name, loaded_weight
continue
targets = (key,)
for layer_idx, expert_id in targets:
if layer_idx not in self._expert_substitutions:
continue
param_name, substitution = self._expert_substitutions[layer_idx]
param = substitution.values
param.weight_loader(param, loaded_weight, expert_id)
loaded_substitutions.add(param_name)
loaded = super().load_weights(remaining_weights())
if loaded is not None:
loaded.update(loaded_substitutions)
return loaded
def process_weights_after_loading(self) -> None:
for param_name, substitution in self._expert_substitutions.values():
substitution.validate_loaded_values(param_name)
finalize = getattr(super(), "process_weights_after_loading", None)
if finalize is not None:
finalize()
ExpertSubstitutionModel.__name__ = model_cls.__name__
ExpertSubstitutionModel.__qualname__ = model_cls.__qualname__
return ExpertSubstitutionModel