[Bugfix] Bind routed-experts capture to the MoE layer, not the kernel (#59455)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen02
Claude Opus 5.5
parent
0979892992
commit
4c4003bb5f
@@ -295,6 +295,7 @@ def test_flashinfer_per_tensor_moe_fp8_no_graph(
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=True,
|
||||
routed_scaling_factor=1.0,
|
||||
routing_sink=None,
|
||||
)
|
||||
|
||||
check_accuracy(
|
||||
@@ -968,6 +969,7 @@ def test_trtllm_mxfp8_minimax2_routing_applies_routed_scale(m: int, workspace_in
|
||||
router_logits=logits,
|
||||
e_score_correction_bias=bias,
|
||||
routed_scaling_factor=scale,
|
||||
routing_sink=None,
|
||||
**common,
|
||||
)
|
||||
topk_weights, topk_ids = fused_topk_bias(
|
||||
|
||||
@@ -781,6 +781,7 @@ def test_trtllm_mxfp4_deferred_finalize(
|
||||
global_num_experts=num_experts,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
routing_sink=None,
|
||||
),
|
||||
)
|
||||
return
|
||||
@@ -1780,6 +1781,7 @@ def test_rocm_mxfp4_moe_oracle(
|
||||
global_num_experts=num_experts,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
routing_sink=None,
|
||||
)
|
||||
else:
|
||||
# Modular impl uses topk_weights and topk_ids
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -11,8 +11,16 @@ from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CompilationMode
|
||||
from vllm.distributed.eplb.eplb_state import EplbLayerState
|
||||
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_int4_moe import CPUExpertsInt4
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import (
|
||||
FusedMoEKernelMonolithicImpl,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.no_dp_ep import (
|
||||
MoEPrepareAndFinalizeNoDPEPMonolithic,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
RoutedExpertsCapturer,
|
||||
RoutedExpertsSink,
|
||||
bind_routed_experts_capturer,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.base_router import BaseRouter
|
||||
@@ -174,45 +182,139 @@ def test_public_binding_supports_direct_capture_source():
|
||||
assert calls == [(3, topk_ids)]
|
||||
|
||||
|
||||
def test_public_binding_rejects_monolithic_without_replay_support(monkeypatch):
|
||||
class DummyFusedMoE:
|
||||
def __init__(self):
|
||||
self.layer_id = 3
|
||||
self.router = _make_router()
|
||||
# Use a concrete monolithic expert and override its capability
|
||||
# instead of instantiating the abstract base class directly.
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
CPUExpertsFp8,
|
||||
)
|
||||
_SINK_CONFIG = SimpleNamespace(
|
||||
use_ep=False,
|
||||
dp_size=1,
|
||||
ep_size=1,
|
||||
max_num_tokens=4,
|
||||
experts_per_token=2,
|
||||
device="cpu",
|
||||
)
|
||||
|
||||
fused_experts = CPUExpertsFp8.__new__(CPUExpertsFp8)
|
||||
self.routed_experts = types.SimpleNamespace(
|
||||
quant_method=types.SimpleNamespace(
|
||||
is_monolithic=True,
|
||||
moe_kernel=types.SimpleNamespace(
|
||||
impl=types.SimpleNamespace(fused_experts=fused_experts)
|
||||
),
|
||||
)
|
||||
)
|
||||
self._quant_method = self.routed_experts.quant_method
|
||||
self._quant_method.moe_kernel.impl.fused_experts = fused_experts
|
||||
fused_experts.supports_routing_replay_capture = lambda: False
|
||||
|
||||
class DummyCapturer:
|
||||
def capture(self, layer_id, topk_ids):
|
||||
pass
|
||||
class _ToyMonolithicExperts(CPUExpertsInt4):
|
||||
"""A monolithic kernel instance, as a weight reload rebuilds it: routes
|
||||
token t to experts (t + offset, t + offset + 1)."""
|
||||
|
||||
dummy_module = DummyFusedMoE()
|
||||
moe_config = _SINK_CONFIG
|
||||
quant_config = None
|
||||
|
||||
def __init__(self, offset: int = 0, supports_capture: bool = True):
|
||||
self.offset = offset
|
||||
self.supports_capture = supports_capture
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
return True
|
||||
|
||||
def supports_routing_replay_capture(self) -> bool:
|
||||
return self.supports_capture
|
||||
|
||||
def apply(self, hidden_states, *args, routing_replay_out=None, **kwargs):
|
||||
if routing_replay_out is not None:
|
||||
t = torch.arange(hidden_states.shape[0])
|
||||
routing_replay_out[: len(t)] = torch.stack([t, t + 1], 1) + self.offset
|
||||
return hidden_states
|
||||
|
||||
|
||||
class _MonolithicMoE:
|
||||
"""The parts of a MoE layer that capture binding reads."""
|
||||
|
||||
layer_id = 5
|
||||
|
||||
def __init__(self, experts: _ToyMonolithicExperts):
|
||||
self.router = _make_router()
|
||||
kernel = SimpleNamespace(impl=SimpleNamespace(fused_experts=experts))
|
||||
self._quant_method = SimpleNamespace(is_monolithic=True, moe_kernel=kernel)
|
||||
self.routed_experts = SimpleNamespace(
|
||||
quant_method=self._quant_method, routing_sink=None
|
||||
)
|
||||
|
||||
|
||||
def _bind_monolithic(
|
||||
monkeypatch, experts: _ToyMonolithicExperts, capture
|
||||
) -> _MonolithicMoE:
|
||||
import vllm.model_executor.layers.fused_moe.layer as fused_moe_layer
|
||||
|
||||
monkeypatch.setattr(fused_moe_layer, "MoERunner", DummyFusedMoE)
|
||||
monkeypatch.setattr(fused_moe_layer, "MoERunner", _MonolithicMoE)
|
||||
layer = _MonolithicMoE(experts)
|
||||
bind_routed_experts_capturer(
|
||||
SimpleNamespace(modules=lambda: [layer]), SimpleNamespace(capture=capture)
|
||||
)
|
||||
return layer
|
||||
|
||||
|
||||
def test_public_binding_gives_monolithic_layers_a_sink(monkeypatch):
|
||||
"""A monolithic kernel routes internally, so binding gives the layer, which
|
||||
outlives kernel rebuilds, a sink that captures under its layer id."""
|
||||
calls = []
|
||||
layer = _bind_monolithic(
|
||||
monkeypatch, _ToyMonolithicExperts(), lambda *args: calls.append(args)
|
||||
)
|
||||
ids = torch.tensor([[1, 2]])
|
||||
layer.routed_experts.routing_sink.capture_fn(ids)
|
||||
assert calls == [(_MonolithicMoE.layer_id, ids)]
|
||||
|
||||
|
||||
def test_public_binding_rejects_monolithic_without_replay_support(monkeypatch):
|
||||
with pytest.raises(ValueError, match="monolithic MoE kernel"):
|
||||
bind_routed_experts_capturer(
|
||||
types.SimpleNamespace(modules=lambda: [dummy_module]), DummyCapturer()
|
||||
_bind_monolithic(
|
||||
monkeypatch, _ToyMonolithicExperts(supports_capture=False), Mock()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("use_ep", "dp_size", "ep_size", "rows"),
|
||||
[(False, 1, 1, 4), (False, 2, 4, 8), (True, 2, 4, 16)],
|
||||
ids=["single-rank", "dp", "ep"],
|
||||
)
|
||||
def test_routed_experts_sink_holds_a_dispatched_batch(use_ep, dp_size, ep_size, rows):
|
||||
"""One int16 buffer, allocated once, large enough for the batch the kernel
|
||||
sees after a DP or EP gather."""
|
||||
config = SimpleNamespace(
|
||||
**{
|
||||
**vars(_SINK_CONFIG),
|
||||
"use_ep": use_ep,
|
||||
"dp_size": dp_size,
|
||||
"ep_size": ep_size,
|
||||
}
|
||||
)
|
||||
sink = RoutedExpertsSink(config, Mock())
|
||||
|
||||
assert sink.buffer.shape == (rows, 2) and sink.buffer.dtype == torch.int16
|
||||
|
||||
|
||||
def test_monolithic_capture_survives_kernel_rebuilds():
|
||||
"""The bug in #59449: a weight reload rebuilds the kernel. Through the
|
||||
production kernel path, the layer's sink captures what the current kernel
|
||||
routed, before and after a rebuild, and a rebuilt kernel that cannot
|
||||
capture is refused instead of leaving stale rows."""
|
||||
captured = []
|
||||
sink = RoutedExpertsSink(_SINK_CONFIG, lambda ids: captured.append(ids.tolist()))
|
||||
|
||||
def forward(experts: _ToyMonolithicExperts) -> None:
|
||||
kernel = FusedMoEKernelMonolithicImpl(
|
||||
MoEPrepareAndFinalizeNoDPEPMonolithic(), experts
|
||||
)
|
||||
kernel.apply(
|
||||
torch.zeros(3, 8),
|
||||
w1=None,
|
||||
w2=None,
|
||||
router_logits=torch.zeros(3, 16),
|
||||
activation=None,
|
||||
global_num_experts=16,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
routing_sink=sink,
|
||||
)
|
||||
|
||||
for offset in (0, 7): # the kernel before and after a reload
|
||||
forward(_ToyMonolithicExperts(offset))
|
||||
assert captured[-1] == [[t + offset, t + offset + 1] for t in range(3)]
|
||||
with pytest.raises(ValueError, match="not supported"):
|
||||
forward(_ToyMonolithicExperts(supports_capture=False))
|
||||
|
||||
|
||||
def test_routed_experts_capturer_single_dp_no_metadata():
|
||||
"""dp_metadata is None: capture writes the full topk_ids rows."""
|
||||
capturer = _capturer_with_buffer(dp_rank=0)
|
||||
|
||||
@@ -374,6 +374,7 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
routing_method = self.moe_config.routing_method
|
||||
if routing_method == RoutingMethodType.DeepseekV4:
|
||||
|
||||
@@ -289,6 +289,7 @@ class AiterW4A8ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert self.moe_config.intermediate_size_per_partition_unpadded is not None
|
||||
assert self.moe_config.hidden_dim_unpadded is not None
|
||||
|
||||
@@ -198,6 +198,7 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Apply the monolithic 4-bit INT MoE forward pass.
|
||||
|
||||
@@ -215,6 +216,7 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
|
||||
e_score_correction_bias: Bias for expert scores
|
||||
routed_scaling_factor: Scaling factor for routing
|
||||
topk_group: Group size for topk
|
||||
routing_replay_out: Unused; this kernel does not capture routing
|
||||
|
||||
Returns:
|
||||
Output tensor after MoE computation
|
||||
|
||||
@@ -1580,6 +1580,7 @@ class OAITritonMxfp4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return triton_kernel_moe_forward(
|
||||
hidden_states=hidden_states,
|
||||
|
||||
@@ -272,6 +272,7 @@ class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonol
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
import flashinfer
|
||||
|
||||
@@ -282,10 +283,6 @@ class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonol
|
||||
# dummy 0-token forward has to keep the finalized (empty) form.
|
||||
defer = self.moe_config.should_defer_moe_finalize(num_tokens)
|
||||
|
||||
routing_replay_out = self._maybe_make_routing_replay_buffer(
|
||||
num_tokens=num_tokens,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
flashinfer_output = flashinfer.fused_moe.trtllm_bf16_moe(
|
||||
routing_logits=router_logits,
|
||||
routing_bias=e_score_correction_bias,
|
||||
@@ -312,5 +309,4 @@ class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonol
|
||||
num_tokens=num_tokens,
|
||||
top_k=self.topk,
|
||||
)
|
||||
self._maybe_dispatch_routing_replay(routing_replay_out, num_tokens=num_tokens)
|
||||
return routed_output
|
||||
|
||||
@@ -450,6 +450,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
import flashinfer
|
||||
from flashinfer.fused_moe import Fp8QuantizationType, WeightLayout
|
||||
@@ -487,11 +488,6 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
n_group = num_expert_group or 0
|
||||
selected_topk_group = topk_group or 0
|
||||
|
||||
routing_replay_out = self._maybe_make_routing_replay_buffer(
|
||||
num_tokens=hidden_states.shape[0],
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
kwargs = dict(
|
||||
routing_logits=router_logits,
|
||||
routing_bias=e_score_correction_bias,
|
||||
@@ -522,9 +518,6 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
if is_mxfp8 or activation == MoEActivation.RELU2_NO_MUL:
|
||||
kwargs["activation_type"] = activation_type
|
||||
result = flashinfer.fused_moe.trtllm_fp8_block_scale_moe(**kwargs)
|
||||
self._maybe_dispatch_routing_replay(
|
||||
routing_replay_out, num_tokens=hidden_states.shape[0]
|
||||
)
|
||||
return result
|
||||
|
||||
def _apply_per_tensor(
|
||||
@@ -543,6 +536,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
# Delay import for non-CUDA.
|
||||
import flashinfer
|
||||
@@ -558,11 +552,6 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
else:
|
||||
assert not apply_router_weight_on_input
|
||||
|
||||
routing_replay_out = self._maybe_make_routing_replay_buffer(
|
||||
num_tokens=hidden_states.shape[0],
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
out = flashinfer.fused_moe.trtllm_fp8_per_tensor_scale_moe(
|
||||
routing_logits=router_logits,
|
||||
routing_bias=e_score_correction_bias,
|
||||
@@ -586,9 +575,6 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
|
||||
routing_replay_out=routing_replay_out,
|
||||
)
|
||||
self._maybe_dispatch_routing_replay(
|
||||
routing_replay_out, num_tokens=hidden_states.shape[0]
|
||||
)
|
||||
return out
|
||||
|
||||
def apply(
|
||||
@@ -607,6 +593,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if self.quant_config.block_shape is not None:
|
||||
return self._apply_block_scale(
|
||||
@@ -623,6 +610,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
topk_group=topk_group,
|
||||
routing_replay_out=routing_replay_out,
|
||||
)
|
||||
elif self.quant_config.is_per_tensor:
|
||||
return self._apply_per_tensor(
|
||||
@@ -638,6 +626,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
num_expert_group=num_expert_group,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
routing_replay_out=routing_replay_out,
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
|
||||
@@ -215,6 +215,7 @@ class TrtLlmMxfp4ExpertsMonolithic(
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
from flashinfer import trtllm_fp4_block_scale_moe
|
||||
|
||||
@@ -236,10 +237,6 @@ class TrtLlmMxfp4ExpertsMonolithic(
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
routing_replay_out = self._maybe_make_routing_replay_buffer(
|
||||
num_tokens=num_tokens,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
flashinfer_output = trtllm_fp4_block_scale_moe(
|
||||
routing_logits=router_logits,
|
||||
routing_bias=e_score_correction_bias,
|
||||
@@ -279,7 +276,6 @@ class TrtLlmMxfp4ExpertsMonolithic(
|
||||
top_k=self.topk,
|
||||
finalized_output=finalized_output,
|
||||
)
|
||||
self._maybe_dispatch_routing_replay(routing_replay_out, num_tokens=num_tokens)
|
||||
return routed_output
|
||||
|
||||
|
||||
|
||||
@@ -139,6 +139,7 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_mxint4_moe import ( # noqa: E501
|
||||
flashinfer_trtllm_mxint4_moe,
|
||||
@@ -147,10 +148,6 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
assert self.w1_scale is not None
|
||||
assert self.w2_scale is not None
|
||||
|
||||
routing_replay_out = self._maybe_make_routing_replay_buffer(
|
||||
num_tokens=hidden_states.shape[0],
|
||||
device=hidden_states.device,
|
||||
)
|
||||
result = flashinfer_trtllm_mxint4_moe(
|
||||
x=hidden_states,
|
||||
router_logits=router_logits,
|
||||
@@ -169,7 +166,4 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
routing_method_type=self.routing_method,
|
||||
routing_replay_out=routing_replay_out,
|
||||
)
|
||||
self._maybe_dispatch_routing_replay(
|
||||
routing_replay_out, num_tokens=hidden_states.shape[0]
|
||||
)
|
||||
return result
|
||||
|
||||
@@ -515,6 +515,7 @@ class TrtLlmNvFp4ExpertsMonolithic(
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
import flashinfer
|
||||
|
||||
@@ -545,10 +546,6 @@ class TrtLlmNvFp4ExpertsMonolithic(
|
||||
# dummy 0-token forward has to keep the finalized (empty) form.
|
||||
defer = self.moe_config.should_defer_moe_finalize(num_tokens)
|
||||
|
||||
routing_replay_out = self._maybe_make_routing_replay_buffer(
|
||||
num_tokens=num_tokens,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
# Invoke kernel.
|
||||
# NOTE: Activation padding and output
|
||||
# truncation are handled by the MoE runner's
|
||||
@@ -592,5 +589,4 @@ class TrtLlmNvFp4ExpertsMonolithic(
|
||||
num_tokens=num_tokens,
|
||||
top_k=self.topk,
|
||||
)
|
||||
self._maybe_dispatch_routing_replay(routing_replay_out, num_tokens=num_tokens)
|
||||
return routed_output
|
||||
|
||||
@@ -151,6 +151,7 @@ class ZentorchExpertsInt4(mk.FusedMoEExpertsMonolithic):
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if apply_router_weight_on_input:
|
||||
raise NotImplementedError(
|
||||
|
||||
@@ -23,6 +23,9 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
RoutedExpertsSink,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.runner.shared_experts import (
|
||||
SharedExperts,
|
||||
SharedExpertsOrder,
|
||||
@@ -979,9 +982,6 @@ class FusedMoEExpertsMonolithic(FusedMoEExperts):
|
||||
def is_monolithic() -> bool:
|
||||
return True
|
||||
|
||||
routing_replay_capture_fn: Callable[[torch.Tensor], None] | None = None
|
||||
_routing_replay_buffer: torch.Tensor | None = None
|
||||
|
||||
def supports_routing_replay_capture(self) -> bool:
|
||||
"""Whether this expert supports routing replay capture.
|
||||
|
||||
@@ -990,53 +990,6 @@ class FusedMoEExpertsMonolithic(FusedMoEExperts):
|
||||
"""
|
||||
return False
|
||||
|
||||
def set_capture_fn(
|
||||
self,
|
||||
capture_fn: Callable[[torch.Tensor], None] | None,
|
||||
) -> None:
|
||||
self.routing_replay_capture_fn = capture_fn
|
||||
if capture_fn is None:
|
||||
self._routing_replay_buffer = None
|
||||
return
|
||||
# Allocate for per-rank batches gathered across the DP or EP group.
|
||||
dispatch_group_size = (
|
||||
self.moe_config.ep_size
|
||||
if self.moe_config.use_ep
|
||||
else self.moe_config.dp_size
|
||||
)
|
||||
max_num_replay_tokens = self.moe_config.max_num_tokens * dispatch_group_size
|
||||
self._routing_replay_buffer = torch.empty(
|
||||
(max_num_replay_tokens, self.moe_config.experts_per_token),
|
||||
dtype=torch.int16,
|
||||
device=self.moe_config.device,
|
||||
)
|
||||
|
||||
def _maybe_make_routing_replay_buffer(
|
||||
self,
|
||||
num_tokens: int,
|
||||
device: torch.device,
|
||||
) -> torch.Tensor | None:
|
||||
if self.routing_replay_capture_fn is None:
|
||||
return None
|
||||
buf = self._routing_replay_buffer
|
||||
assert buf is not None
|
||||
if buf.shape[0] < num_tokens or buf.device != device:
|
||||
raise ValueError(
|
||||
"Routing replay buffer was initialized for "
|
||||
f"{buf.shape[0]} tokens on {buf.device}, but the kernel "
|
||||
f"received {num_tokens} tokens on {device}."
|
||||
)
|
||||
return buf
|
||||
|
||||
def _maybe_dispatch_routing_replay(
|
||||
self,
|
||||
routing_replay_out: torch.Tensor | None,
|
||||
num_tokens: int,
|
||||
) -> None:
|
||||
if routing_replay_out is None or self.routing_replay_capture_fn is None:
|
||||
return
|
||||
self.routing_replay_capture_fn(routing_replay_out[:num_tokens])
|
||||
|
||||
def apply(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -1053,10 +1006,14 @@ class FusedMoEExpertsMonolithic(FusedMoEExperts):
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
"""Same as ``FusedMoEExperts.apply``, except uses router_logits as opposed
|
||||
to the topk_ids and topk_weights. This is useful for kernels
|
||||
with fused router and fused_experts (e.g. FLASHINFER_TRTLLM).
|
||||
|
||||
Kernels that ``supports_routing_replay_capture`` write each token's
|
||||
routed expert ids into the leading rows of ``routing_replay_out``.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -1563,6 +1520,8 @@ class FusedMoEKernelMonolithicImpl:
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
*,
|
||||
routing_sink: RoutedExpertsSink | None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
"""Same as forward(), except uses router_logits as opposed
|
||||
to the topk_ids and topk_weights. This is used for kernels
|
||||
@@ -1575,6 +1534,15 @@ class FusedMoEKernelMonolithicImpl:
|
||||
defer_input_quant=self.fused_experts.expects_unquantized_inputs,
|
||||
)
|
||||
|
||||
routing_replay_out = None
|
||||
if routing_sink is not None:
|
||||
# Checked per call: a weight reload may rebuild a different kernel.
|
||||
if not self.fused_experts.supports_routing_replay_capture():
|
||||
raise ValueError(
|
||||
"Routed-experts capture is not supported with monolithic MoE "
|
||||
f"kernel {type(self.fused_experts).__name__}."
|
||||
)
|
||||
routing_replay_out = routing_sink.buffer[: len(a1q)]
|
||||
fused_out = self.fused_experts.apply(
|
||||
hidden_states=a1q,
|
||||
w1=w1,
|
||||
@@ -1590,7 +1558,10 @@ class FusedMoEKernelMonolithicImpl:
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
topk_group=topk_group,
|
||||
routing_replay_out=routing_replay_out,
|
||||
)
|
||||
if routing_sink is not None:
|
||||
routing_sink.capture_fn(routing_replay_out)
|
||||
|
||||
if isinstance(fused_out, UnfinalizedMoEOutput):
|
||||
if not self.prepare_finalize.supports_deferred_moe_finalize():
|
||||
@@ -1698,6 +1669,8 @@ class FusedMoEKernel:
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
*,
|
||||
routing_sink: RoutedExpertsSink | None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert isinstance(self.impl, FusedMoEKernelMonolithicImpl)
|
||||
return self.impl.apply(
|
||||
@@ -1713,6 +1686,7 @@ class FusedMoEKernel:
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
topk_group=topk_group,
|
||||
routing_sink=routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
@@ -20,6 +20,9 @@ from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
|
||||
FusedMoEMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
RoutedExpertsSink,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
@@ -138,6 +141,8 @@ class RoutedExperts(PluggableLayer):
|
||||
self.swiglu_alpha = swiglu_alpha
|
||||
self.swiglu_beta = swiglu_beta
|
||||
self.e_score_correction_bias = e_score_correction_bias
|
||||
# Set by bind_routed_experts_capturer for monolithic kernels.
|
||||
self.routing_sink: RoutedExpertsSink | None = None
|
||||
self.apply_router_weight_on_input = apply_router_weight_on_input
|
||||
# End random parameters
|
||||
self._loaded_expert_biases: set[str] = set()
|
||||
|
||||
@@ -15,6 +15,7 @@ import torch
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.parallel_state import get_tp_group
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -186,6 +187,23 @@ class RoutedExpertsCapturer:
|
||||
return self.device_buffer[:num_tokens].to(self.output_dtype)
|
||||
|
||||
|
||||
class RoutedExpertsSink:
|
||||
"""Layer-owned buffer and callback for the expert ids a monolithic kernel
|
||||
routes to; it outlives kernel rebuilds on weight reload."""
|
||||
|
||||
def __init__(
|
||||
self, moe_config: FusedMoEConfig, capture_fn: Callable[[torch.Tensor], None]
|
||||
) -> None:
|
||||
# Sized for per-rank batches gathered across the DP or EP group.
|
||||
group_size = moe_config.ep_size if moe_config.use_ep else moe_config.dp_size
|
||||
self.buffer = torch.empty(
|
||||
(moe_config.max_num_tokens * group_size, moe_config.experts_per_token),
|
||||
dtype=torch.int16,
|
||||
device=moe_config.device,
|
||||
)
|
||||
self.capture_fn = capture_fn
|
||||
|
||||
|
||||
def bind_routed_experts_capturer(
|
||||
model: torch.nn.Module,
|
||||
capturer: RoutedExpertsCapturer,
|
||||
@@ -219,7 +237,9 @@ def bind_routed_experts_capturer(
|
||||
"Routed-experts capture is not supported with monolithic "
|
||||
f"MoE kernel {type(fused_experts).__name__}."
|
||||
)
|
||||
fused_experts.set_capture_fn(capture_fn)
|
||||
module.routed_experts.routing_sink = RoutedExpertsSink(
|
||||
fused_experts.moe_config, capture_fn
|
||||
)
|
||||
num_bound += 1
|
||||
elif isinstance(module.router, BaseRouter):
|
||||
module.router.set_capture_fn(capture_fn)
|
||||
|
||||
@@ -362,4 +362,5 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
@@ -790,6 +790,7 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -779,4 +779,5 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
+1
@@ -293,6 +293,7 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
+1
@@ -287,6 +287,7 @@ class CompressedTensorsW4A8Int8MoEMethod(CompressedTensorsMoEMethod):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
+1
@@ -366,6 +366,7 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
+1
@@ -291,4 +291,5 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
+1
@@ -179,6 +179,7 @@ class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
+1
@@ -528,6 +528,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
@@ -818,6 +818,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
@@ -680,6 +680,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
@@ -1092,6 +1093,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
@@ -1461,6 +1463,7 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
@@ -531,6 +531,7 @@ class MoeWNA16Method(FusedMoEMethodBase):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -472,6 +472,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
|
||||
@@ -900,4 +901,5 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
@@ -156,6 +156,7 @@ class OnlineMoEMethodBase(FusedMoEMethodBase):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
@@ -1980,6 +1980,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user