[Bugfix] Bind routed-experts capture to the MoE layer, not the kernel (#59455)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Eric Tang
2026-10-02 15:50:38 +00:00
committed by GitHub
co-authored by aoshen02 Claude Opus 5.5
parent 0979892992
commit 4c4003bb5f
32 changed files with 423 additions and 898 deletions
+2
View File
@@ -295,6 +295,7 @@ def test_flashinfer_per_tensor_moe_fp8_no_graph(
expert_map=None,
apply_router_weight_on_input=True,
routed_scaling_factor=1.0,
routing_sink=None,
)
check_accuracy(
@@ -968,6 +969,7 @@ def test_trtllm_mxfp8_minimax2_routing_applies_routed_scale(m: int, workspace_in
router_logits=logits,
e_score_correction_bias=bias,
routed_scaling_factor=scale,
routing_sink=None,
**common,
)
topk_weights, topk_ids = fused_topk_bias(
+2
View File
@@ -781,6 +781,7 @@ def test_trtllm_mxfp4_deferred_finalize(
global_num_experts=num_experts,
expert_map=None,
apply_router_weight_on_input=False,
routing_sink=None,
),
)
return
@@ -1780,6 +1781,7 @@ def test_rocm_mxfp4_moe_oracle(
global_num_experts=num_experts,
expert_map=None,
apply_router_weight_on_input=False,
routing_sink=None,
)
else:
# Modular impl uses topk_weights and topk_ids
File diff suppressed because it is too large Load Diff
@@ -11,8 +11,16 @@ from vllm.config import VllmConfig
from vllm.config.compilation import CompilationMode
from vllm.distributed.eplb.eplb_state import EplbLayerState
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
from vllm.model_executor.layers.fused_moe.experts.cpu_int4_moe import CPUExpertsInt4
from vllm.model_executor.layers.fused_moe.modular_kernel import (
FusedMoEKernelMonolithicImpl,
)
from vllm.model_executor.layers.fused_moe.prepare_finalize.no_dp_ep import (
MoEPrepareAndFinalizeNoDPEPMonolithic,
)
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
RoutedExpertsCapturer,
RoutedExpertsSink,
bind_routed_experts_capturer,
)
from vllm.model_executor.layers.fused_moe.router.base_router import BaseRouter
@@ -174,45 +182,139 @@ def test_public_binding_supports_direct_capture_source():
assert calls == [(3, topk_ids)]
def test_public_binding_rejects_monolithic_without_replay_support(monkeypatch):
class DummyFusedMoE:
def __init__(self):
self.layer_id = 3
self.router = _make_router()
# Use a concrete monolithic expert and override its capability
# instead of instantiating the abstract base class directly.
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
CPUExpertsFp8,
)
_SINK_CONFIG = SimpleNamespace(
use_ep=False,
dp_size=1,
ep_size=1,
max_num_tokens=4,
experts_per_token=2,
device="cpu",
)
fused_experts = CPUExpertsFp8.__new__(CPUExpertsFp8)
self.routed_experts = types.SimpleNamespace(
quant_method=types.SimpleNamespace(
is_monolithic=True,
moe_kernel=types.SimpleNamespace(
impl=types.SimpleNamespace(fused_experts=fused_experts)
),
)
)
self._quant_method = self.routed_experts.quant_method
self._quant_method.moe_kernel.impl.fused_experts = fused_experts
fused_experts.supports_routing_replay_capture = lambda: False
class DummyCapturer:
def capture(self, layer_id, topk_ids):
pass
class _ToyMonolithicExperts(CPUExpertsInt4):
"""A monolithic kernel instance, as a weight reload rebuilds it: routes
token t to experts (t + offset, t + offset + 1)."""
dummy_module = DummyFusedMoE()
moe_config = _SINK_CONFIG
quant_config = None
def __init__(self, offset: int = 0, supports_capture: bool = True):
self.offset = offset
self.supports_capture = supports_capture
@property
def expects_unquantized_inputs(self) -> bool:
return True
def supports_routing_replay_capture(self) -> bool:
return self.supports_capture
def apply(self, hidden_states, *args, routing_replay_out=None, **kwargs):
if routing_replay_out is not None:
t = torch.arange(hidden_states.shape[0])
routing_replay_out[: len(t)] = torch.stack([t, t + 1], 1) + self.offset
return hidden_states
class _MonolithicMoE:
"""The parts of a MoE layer that capture binding reads."""
layer_id = 5
def __init__(self, experts: _ToyMonolithicExperts):
self.router = _make_router()
kernel = SimpleNamespace(impl=SimpleNamespace(fused_experts=experts))
self._quant_method = SimpleNamespace(is_monolithic=True, moe_kernel=kernel)
self.routed_experts = SimpleNamespace(
quant_method=self._quant_method, routing_sink=None
)
def _bind_monolithic(
monkeypatch, experts: _ToyMonolithicExperts, capture
) -> _MonolithicMoE:
import vllm.model_executor.layers.fused_moe.layer as fused_moe_layer
monkeypatch.setattr(fused_moe_layer, "MoERunner", DummyFusedMoE)
monkeypatch.setattr(fused_moe_layer, "MoERunner", _MonolithicMoE)
layer = _MonolithicMoE(experts)
bind_routed_experts_capturer(
SimpleNamespace(modules=lambda: [layer]), SimpleNamespace(capture=capture)
)
return layer
def test_public_binding_gives_monolithic_layers_a_sink(monkeypatch):
"""A monolithic kernel routes internally, so binding gives the layer, which
outlives kernel rebuilds, a sink that captures under its layer id."""
calls = []
layer = _bind_monolithic(
monkeypatch, _ToyMonolithicExperts(), lambda *args: calls.append(args)
)
ids = torch.tensor([[1, 2]])
layer.routed_experts.routing_sink.capture_fn(ids)
assert calls == [(_MonolithicMoE.layer_id, ids)]
def test_public_binding_rejects_monolithic_without_replay_support(monkeypatch):
with pytest.raises(ValueError, match="monolithic MoE kernel"):
bind_routed_experts_capturer(
types.SimpleNamespace(modules=lambda: [dummy_module]), DummyCapturer()
_bind_monolithic(
monkeypatch, _ToyMonolithicExperts(supports_capture=False), Mock()
)
@pytest.mark.parametrize(
("use_ep", "dp_size", "ep_size", "rows"),
[(False, 1, 1, 4), (False, 2, 4, 8), (True, 2, 4, 16)],
ids=["single-rank", "dp", "ep"],
)
def test_routed_experts_sink_holds_a_dispatched_batch(use_ep, dp_size, ep_size, rows):
"""One int16 buffer, allocated once, large enough for the batch the kernel
sees after a DP or EP gather."""
config = SimpleNamespace(
**{
**vars(_SINK_CONFIG),
"use_ep": use_ep,
"dp_size": dp_size,
"ep_size": ep_size,
}
)
sink = RoutedExpertsSink(config, Mock())
assert sink.buffer.shape == (rows, 2) and sink.buffer.dtype == torch.int16
def test_monolithic_capture_survives_kernel_rebuilds():
"""The bug in #59449: a weight reload rebuilds the kernel. Through the
production kernel path, the layer's sink captures what the current kernel
routed, before and after a rebuild, and a rebuilt kernel that cannot
capture is refused instead of leaving stale rows."""
captured = []
sink = RoutedExpertsSink(_SINK_CONFIG, lambda ids: captured.append(ids.tolist()))
def forward(experts: _ToyMonolithicExperts) -> None:
kernel = FusedMoEKernelMonolithicImpl(
MoEPrepareAndFinalizeNoDPEPMonolithic(), experts
)
kernel.apply(
torch.zeros(3, 8),
w1=None,
w2=None,
router_logits=torch.zeros(3, 16),
activation=None,
global_num_experts=16,
expert_map=None,
apply_router_weight_on_input=False,
routing_sink=sink,
)
for offset in (0, 7): # the kernel before and after a reload
forward(_ToyMonolithicExperts(offset))
assert captured[-1] == [[t + offset, t + offset + 1] for t in range(3)]
with pytest.raises(ValueError, match="not supported"):
forward(_ToyMonolithicExperts(supports_capture=False))
def test_routed_experts_capturer_single_dp_no_metadata():
"""dp_metadata is None: capture writes the full topk_ids rows."""
capturer = _capturer_with_buffer(dp_rank=0)
@@ -374,6 +374,7 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor:
routing_method = self.moe_config.routing_method
if routing_method == RoutingMethodType.DeepseekV4:
@@ -289,6 +289,7 @@ class AiterW4A8ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor:
assert self.moe_config.intermediate_size_per_partition_unpadded is not None
assert self.moe_config.hidden_dim_unpadded is not None
@@ -198,6 +198,7 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor:
"""Apply the monolithic 4-bit INT MoE forward pass.
@@ -215,6 +216,7 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
e_score_correction_bias: Bias for expert scores
routed_scaling_factor: Scaling factor for routing
topk_group: Group size for topk
routing_replay_out: Unused; this kernel does not capture routing
Returns:
Output tensor after MoE computation
@@ -1580,6 +1580,7 @@ class OAITritonMxfp4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor:
return triton_kernel_moe_forward(
hidden_states=hidden_states,
@@ -272,6 +272,7 @@ class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonol
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor | UnfinalizedMoEOutput:
import flashinfer
@@ -282,10 +283,6 @@ class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonol
# dummy 0-token forward has to keep the finalized (empty) form.
defer = self.moe_config.should_defer_moe_finalize(num_tokens)
routing_replay_out = self._maybe_make_routing_replay_buffer(
num_tokens=num_tokens,
device=hidden_states.device,
)
flashinfer_output = flashinfer.fused_moe.trtllm_bf16_moe(
routing_logits=router_logits,
routing_bias=e_score_correction_bias,
@@ -312,5 +309,4 @@ class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonol
num_tokens=num_tokens,
top_k=self.topk,
)
self._maybe_dispatch_routing_replay(routing_replay_out, num_tokens=num_tokens)
return routed_output
@@ -450,6 +450,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor:
import flashinfer
from flashinfer.fused_moe import Fp8QuantizationType, WeightLayout
@@ -487,11 +488,6 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
n_group = num_expert_group or 0
selected_topk_group = topk_group or 0
routing_replay_out = self._maybe_make_routing_replay_buffer(
num_tokens=hidden_states.shape[0],
device=hidden_states.device,
)
kwargs = dict(
routing_logits=router_logits,
routing_bias=e_score_correction_bias,
@@ -522,9 +518,6 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
if is_mxfp8 or activation == MoEActivation.RELU2_NO_MUL:
kwargs["activation_type"] = activation_type
result = flashinfer.fused_moe.trtllm_fp8_block_scale_moe(**kwargs)
self._maybe_dispatch_routing_replay(
routing_replay_out, num_tokens=hidden_states.shape[0]
)
return result
def _apply_per_tensor(
@@ -543,6 +536,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor:
# Delay import for non-CUDA.
import flashinfer
@@ -558,11 +552,6 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
else:
assert not apply_router_weight_on_input
routing_replay_out = self._maybe_make_routing_replay_buffer(
num_tokens=hidden_states.shape[0],
device=hidden_states.device,
)
out = flashinfer.fused_moe.trtllm_fp8_per_tensor_scale_moe(
routing_logits=router_logits,
routing_bias=e_score_correction_bias,
@@ -586,9 +575,6 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
routing_replay_out=routing_replay_out,
)
self._maybe_dispatch_routing_replay(
routing_replay_out, num_tokens=hidden_states.shape[0]
)
return out
def apply(
@@ -607,6 +593,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor:
if self.quant_config.block_shape is not None:
return self._apply_block_scale(
@@ -623,6 +610,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
e_score_correction_bias=e_score_correction_bias,
routed_scaling_factor=routed_scaling_factor,
topk_group=topk_group,
routing_replay_out=routing_replay_out,
)
elif self.quant_config.is_per_tensor:
return self._apply_per_tensor(
@@ -638,6 +626,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
num_expert_group=num_expert_group,
e_score_correction_bias=e_score_correction_bias,
routed_scaling_factor=routed_scaling_factor,
routing_replay_out=routing_replay_out,
)
else:
raise NotImplementedError(
@@ -215,6 +215,7 @@ class TrtLlmMxfp4ExpertsMonolithic(
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor | UnfinalizedMoEOutput:
from flashinfer import trtllm_fp4_block_scale_moe
@@ -236,10 +237,6 @@ class TrtLlmMxfp4ExpertsMonolithic(
device=hidden_states.device,
)
routing_replay_out = self._maybe_make_routing_replay_buffer(
num_tokens=num_tokens,
device=hidden_states.device,
)
flashinfer_output = trtllm_fp4_block_scale_moe(
routing_logits=router_logits,
routing_bias=e_score_correction_bias,
@@ -279,7 +276,6 @@ class TrtLlmMxfp4ExpertsMonolithic(
top_k=self.topk,
finalized_output=finalized_output,
)
self._maybe_dispatch_routing_replay(routing_replay_out, num_tokens=num_tokens)
return routed_output
@@ -139,6 +139,7 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor:
from vllm.model_executor.layers.quantization.utils.flashinfer_mxint4_moe import ( # noqa: E501
flashinfer_trtllm_mxint4_moe,
@@ -147,10 +148,6 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
assert self.w1_scale is not None
assert self.w2_scale is not None
routing_replay_out = self._maybe_make_routing_replay_buffer(
num_tokens=hidden_states.shape[0],
device=hidden_states.device,
)
result = flashinfer_trtllm_mxint4_moe(
x=hidden_states,
router_logits=router_logits,
@@ -169,7 +166,4 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
routing_method_type=self.routing_method,
routing_replay_out=routing_replay_out,
)
self._maybe_dispatch_routing_replay(
routing_replay_out, num_tokens=hidden_states.shape[0]
)
return result
@@ -515,6 +515,7 @@ class TrtLlmNvFp4ExpertsMonolithic(
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor | UnfinalizedMoEOutput:
import flashinfer
@@ -545,10 +546,6 @@ class TrtLlmNvFp4ExpertsMonolithic(
# dummy 0-token forward has to keep the finalized (empty) form.
defer = self.moe_config.should_defer_moe_finalize(num_tokens)
routing_replay_out = self._maybe_make_routing_replay_buffer(
num_tokens=num_tokens,
device=hidden_states.device,
)
# Invoke kernel.
# NOTE: Activation padding and output
# truncation are handled by the MoE runner's
@@ -592,5 +589,4 @@ class TrtLlmNvFp4ExpertsMonolithic(
num_tokens=num_tokens,
top_k=self.topk,
)
self._maybe_dispatch_routing_replay(routing_replay_out, num_tokens=num_tokens)
return routed_output
@@ -151,6 +151,7 @@ class ZentorchExpertsInt4(mk.FusedMoEExpertsMonolithic):
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor:
if apply_router_weight_on_input:
raise NotImplementedError(
@@ -23,6 +23,9 @@ from vllm.model_executor.layers.fused_moe.config import (
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
RoutedExpertsSink,
)
from vllm.model_executor.layers.fused_moe.runner.shared_experts import (
SharedExperts,
SharedExpertsOrder,
@@ -979,9 +982,6 @@ class FusedMoEExpertsMonolithic(FusedMoEExperts):
def is_monolithic() -> bool:
return True
routing_replay_capture_fn: Callable[[torch.Tensor], None] | None = None
_routing_replay_buffer: torch.Tensor | None = None
def supports_routing_replay_capture(self) -> bool:
"""Whether this expert supports routing replay capture.
@@ -990,53 +990,6 @@ class FusedMoEExpertsMonolithic(FusedMoEExperts):
"""
return False
def set_capture_fn(
self,
capture_fn: Callable[[torch.Tensor], None] | None,
) -> None:
self.routing_replay_capture_fn = capture_fn
if capture_fn is None:
self._routing_replay_buffer = None
return
# Allocate for per-rank batches gathered across the DP or EP group.
dispatch_group_size = (
self.moe_config.ep_size
if self.moe_config.use_ep
else self.moe_config.dp_size
)
max_num_replay_tokens = self.moe_config.max_num_tokens * dispatch_group_size
self._routing_replay_buffer = torch.empty(
(max_num_replay_tokens, self.moe_config.experts_per_token),
dtype=torch.int16,
device=self.moe_config.device,
)
def _maybe_make_routing_replay_buffer(
self,
num_tokens: int,
device: torch.device,
) -> torch.Tensor | None:
if self.routing_replay_capture_fn is None:
return None
buf = self._routing_replay_buffer
assert buf is not None
if buf.shape[0] < num_tokens or buf.device != device:
raise ValueError(
"Routing replay buffer was initialized for "
f"{buf.shape[0]} tokens on {buf.device}, but the kernel "
f"received {num_tokens} tokens on {device}."
)
return buf
def _maybe_dispatch_routing_replay(
self,
routing_replay_out: torch.Tensor | None,
num_tokens: int,
) -> None:
if routing_replay_out is None or self.routing_replay_capture_fn is None:
return
self.routing_replay_capture_fn(routing_replay_out[:num_tokens])
def apply(
self,
hidden_states: torch.Tensor,
@@ -1053,10 +1006,14 @@ class FusedMoEExpertsMonolithic(FusedMoEExperts):
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor | UnfinalizedMoEOutput:
"""Same as ``FusedMoEExperts.apply``, except uses router_logits as opposed
to the topk_ids and topk_weights. This is useful for kernels
with fused router and fused_experts (e.g. FLASHINFER_TRTLLM).
Kernels that ``supports_routing_replay_capture`` write each token's
routed expert ids into the leading rows of ``routing_replay_out``.
"""
raise NotImplementedError
@@ -1563,6 +1520,8 @@ class FusedMoEKernelMonolithicImpl:
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
*,
routing_sink: RoutedExpertsSink | None,
) -> torch.Tensor | UnfinalizedMoEOutput:
"""Same as forward(), except uses router_logits as opposed
to the topk_ids and topk_weights. This is used for kernels
@@ -1575,6 +1534,15 @@ class FusedMoEKernelMonolithicImpl:
defer_input_quant=self.fused_experts.expects_unquantized_inputs,
)
routing_replay_out = None
if routing_sink is not None:
# Checked per call: a weight reload may rebuild a different kernel.
if not self.fused_experts.supports_routing_replay_capture():
raise ValueError(
"Routed-experts capture is not supported with monolithic MoE "
f"kernel {type(self.fused_experts).__name__}."
)
routing_replay_out = routing_sink.buffer[: len(a1q)]
fused_out = self.fused_experts.apply(
hidden_states=a1q,
w1=w1,
@@ -1590,7 +1558,10 @@ class FusedMoEKernelMonolithicImpl:
e_score_correction_bias=e_score_correction_bias,
routed_scaling_factor=routed_scaling_factor,
topk_group=topk_group,
routing_replay_out=routing_replay_out,
)
if routing_sink is not None:
routing_sink.capture_fn(routing_replay_out)
if isinstance(fused_out, UnfinalizedMoEOutput):
if not self.prepare_finalize.supports_deferred_moe_finalize():
@@ -1698,6 +1669,8 @@ class FusedMoEKernel:
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
*,
routing_sink: RoutedExpertsSink | None,
) -> torch.Tensor | UnfinalizedMoEOutput:
assert isinstance(self.impl, FusedMoEKernelMonolithicImpl)
return self.impl.apply(
@@ -1713,6 +1686,7 @@ class FusedMoEKernel:
e_score_correction_bias=e_score_correction_bias,
routed_scaling_factor=routed_scaling_factor,
topk_group=topk_group,
routing_sink=routing_sink,
)
def apply(
@@ -20,6 +20,9 @@ from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
FusedMoEMethodBase,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
RoutedExpertsSink,
)
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
UnquantizedFusedMoEMethod,
)
@@ -138,6 +141,8 @@ class RoutedExperts(PluggableLayer):
self.swiglu_alpha = swiglu_alpha
self.swiglu_beta = swiglu_beta
self.e_score_correction_bias = e_score_correction_bias
# Set by bind_routed_experts_capturer for monolithic kernels.
self.routing_sink: RoutedExpertsSink | None = None
self.apply_router_weight_on_input = apply_router_weight_on_input
# End random parameters
self._loaded_expert_biases: set[str] = set()
@@ -15,6 +15,7 @@ import torch
from vllm.config import VllmConfig
from vllm.distributed.parallel_state import get_tp_group
from vllm.forward_context import get_forward_context
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
from vllm.platforms import current_platform
logger = logging.getLogger(__name__)
@@ -186,6 +187,23 @@ class RoutedExpertsCapturer:
return self.device_buffer[:num_tokens].to(self.output_dtype)
class RoutedExpertsSink:
"""Layer-owned buffer and callback for the expert ids a monolithic kernel
routes to; it outlives kernel rebuilds on weight reload."""
def __init__(
self, moe_config: FusedMoEConfig, capture_fn: Callable[[torch.Tensor], None]
) -> None:
# Sized for per-rank batches gathered across the DP or EP group.
group_size = moe_config.ep_size if moe_config.use_ep else moe_config.dp_size
self.buffer = torch.empty(
(moe_config.max_num_tokens * group_size, moe_config.experts_per_token),
dtype=torch.int16,
device=moe_config.device,
)
self.capture_fn = capture_fn
def bind_routed_experts_capturer(
model: torch.nn.Module,
capturer: RoutedExpertsCapturer,
@@ -219,7 +237,9 @@ def bind_routed_experts_capturer(
"Routed-experts capture is not supported with monolithic "
f"MoE kernel {type(fused_experts).__name__}."
)
fused_experts.set_capture_fn(capture_fn)
module.routed_experts.routing_sink = RoutedExpertsSink(
fused_experts.moe_config, capture_fn
)
num_bound += 1
elif isinstance(module.router, BaseRouter):
module.router.set_capture_fn(capture_fn)
@@ -362,4 +362,5 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
@@ -790,6 +790,7 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
@@ -779,4 +779,5 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
@@ -293,6 +293,7 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def apply(
@@ -287,6 +287,7 @@ class CompressedTensorsW4A8Int8MoEMethod(CompressedTensorsMoEMethod):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def apply(
@@ -366,6 +366,7 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def apply(
@@ -291,4 +291,5 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
@@ -179,6 +179,7 @@ class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def apply(
@@ -528,6 +528,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def apply(
@@ -818,6 +818,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def apply(
@@ -680,6 +680,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def apply(
@@ -1092,6 +1093,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def apply(
@@ -1461,6 +1463,7 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def apply(
@@ -531,6 +531,7 @@ class MoeWNA16Method(FusedMoEMethodBase):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
@staticmethod
@@ -472,6 +472,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
@@ -900,4 +901,5 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
@@ -156,6 +156,7 @@ class OnlineMoEMethodBase(FusedMoEMethodBase):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def apply(
@@ -1980,6 +1980,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)