[RL Infra][FlashInfer] Enable router replay output from FlashInfer monolithic MoE kernel (#44214)

Signed-off-by: Xuanyu Zhang <xuanyu.zhang@mistral.ai>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Xuanyu Zhang
2026-07-20 16:45:10 -07:00
committed by GitHub
co-authored by Claude aoshen02 OpenAI Codex
parent 58b2012aa2
commit 97a668152b
11 changed files with 1097 additions and 10 deletions
@@ -34,6 +34,9 @@ class TrtLlmBf16ExpertsBase:
monolithic interfaces.
"""
def supports_routing_replay_capture(self) -> bool:
return True
def __init__(
self,
moe_config: FusedMoEConfig,
@@ -246,7 +249,11 @@ class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonol
assert activation in [MoEActivation.SILU, MoEActivation.RELU2_NO_MUL]
return flashinfer.fused_moe.trtllm_bf16_moe(
routing_replay_out = self._maybe_make_routing_replay_buffer(
num_tokens=hidden_states.shape[0],
device=hidden_states.device,
)
out = flashinfer.fused_moe.trtllm_bf16_moe(
routing_logits=router_logits,
routing_bias=e_score_correction_bias,
hidden_states=hidden_states,
@@ -263,4 +270,9 @@ class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonol
routing_method_type=self.routing_method_type,
activation_type=activation_to_flashinfer_int(activation),
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
routing_replay_out=routing_replay_out,
)
self._maybe_dispatch_routing_replay(
routing_replay_out, num_tokens=hidden_states.shape[0]
)
return out
@@ -265,6 +265,9 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
Fp8 TRTLLM-Gen MoE kernels. Supports monolithic interface.
"""
def supports_routing_replay_capture(self) -> bool:
return True
def __init__(
self,
moe_config: FusedMoEConfig,
@@ -403,6 +406,11 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
n_group = num_expert_group or 0
selected_topk_group = topk_group or 0
routing_replay_out = self._maybe_make_routing_replay_buffer(
num_tokens=hidden_states.shape[0],
device=hidden_states.device,
)
kwargs = dict(
routing_logits=router_logits,
routing_bias=e_score_correction_bias,
@@ -427,11 +435,16 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
use_shuffled_weight=use_shuffled_weight,
weight_layout=weight_layout,
fp8_quantization_type=fp8_quant_type,
routing_replay_out=routing_replay_out,
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
)
if is_mxfp8 or activation == MoEActivation.RELU2_NO_MUL:
kwargs["activation_type"] = activation_type
return flashinfer.fused_moe.trtllm_fp8_block_scale_moe(**kwargs)
result = flashinfer.fused_moe.trtllm_fp8_block_scale_moe(**kwargs)
self._maybe_dispatch_routing_replay(
routing_replay_out, num_tokens=hidden_states.shape[0]
)
return result
def _apply_per_tensor(
self,
@@ -464,6 +477,11 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
else:
assert not apply_router_weight_on_input
routing_replay_out = self._maybe_make_routing_replay_buffer(
num_tokens=hidden_states.shape[0],
device=hidden_states.device,
)
out = flashinfer.fused_moe.trtllm_fp8_per_tensor_scale_moe(
routing_logits=router_logits,
routing_bias=e_score_correction_bias,
@@ -485,6 +503,10 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
routing_method_type=self.routing_method_type,
activation_type=activation_type,
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
routing_replay_out=routing_replay_out,
)
self._maybe_dispatch_routing_replay(
routing_replay_out, num_tokens=hidden_states.shape[0]
)
return out
@@ -122,6 +122,9 @@ class TrtLlmMxfp4ExpertsMonolithic(
Wraps flashinfer.trtllm_fp4_block_scale_moe().
"""
def supports_routing_replay_capture(self) -> bool:
return True
@staticmethod
def _supports_parallel_config(
moe_parallel_config: FusedMoEParallelConfig,
@@ -184,6 +187,10 @@ class TrtLlmMxfp4ExpertsMonolithic(
device=hidden_states.device,
)
routing_replay_out = self._maybe_make_routing_replay_buffer(
num_tokens=hidden_states.shape[0],
device=hidden_states.device,
)
trtllm_fp4_block_scale_moe(
routing_logits=router_logits.to(torch.bfloat16),
routing_bias=None,
@@ -213,8 +220,11 @@ class TrtLlmMxfp4ExpertsMonolithic(
do_finalize=True,
tune_max_num_tokens=max(self.max_capture_size, 1),
output=output,
routing_replay_out=routing_replay_out,
)
self._maybe_dispatch_routing_replay(
routing_replay_out, num_tokens=hidden_states.shape[0]
)
return output
@@ -122,6 +122,9 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
# The kernel handles quantization internally.
return True
def supports_routing_replay_capture(self) -> bool:
return True
def apply(
self,
hidden_states: torch.Tensor,
@@ -144,7 +147,12 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
assert self.w1_scale is not None
assert self.w2_scale is not None
return flashinfer_trtllm_mxint4_moe(
routing_replay_out = self._maybe_make_routing_replay_buffer(
num_tokens=hidden_states.shape[0],
device=hidden_states.device,
)
result = flashinfer_trtllm_mxint4_moe(
x=hidden_states,
router_logits=router_logits,
w13_weight_packed=w1,
@@ -160,4 +168,9 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
topk_group=topk_group,
e_score_correction_bias=e_score_correction_bias,
routing_method_type=self.routing_method,
routing_replay_out=routing_replay_out,
)
self._maybe_dispatch_routing_replay(
routing_replay_out, num_tokens=hidden_states.shape[0]
)
return result
@@ -435,6 +435,9 @@ class TrtLlmNvFp4ExpertsMonolithic(
Monolithic version of the kernel (router + experts).
"""
def supports_routing_replay_capture(self) -> bool:
return True
@staticmethod
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
"""The modular implementation should be used for the Dp/Ep or EPLB case."""
@@ -509,10 +512,14 @@ class TrtLlmNvFp4ExpertsMonolithic(
output1_scale_gate_scalar = self.quant_config.g1_alphas
routing_replay_out = self._maybe_make_routing_replay_buffer(
num_tokens=hidden_states.shape[0],
device=hidden_states.device,
)
# Invoke kernel.
# NOTE: Activation padding and output
# truncation are handled by the MoE runner's
return flashinfer.fused_moe.trtllm_fp4_block_scale_moe(
result = flashinfer.fused_moe.trtllm_fp4_block_scale_moe(
routing_logits=router_logits,
routing_bias=e_score_correction_bias,
hidden_states=hidden_states,
@@ -544,4 +551,9 @@ class TrtLlmNvFp4ExpertsMonolithic(
activation_type=activation_to_flashinfer_int(activation),
per_token_scale=per_token_scale,
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
routing_replay_out=routing_replay_out,
)[0]
self._maybe_dispatch_routing_replay(
routing_replay_out, num_tokens=hidden_states.shape[0]
)
return result
@@ -1000,6 +1000,57 @@ class FusedMoEExpertsMonolithic(FusedMoEExperts):
def is_monolithic() -> bool:
return True
routing_replay_capture_fn: Callable[[torch.Tensor], None] | None = None
_routing_replay_buffer: torch.Tensor | None = None
def supports_routing_replay_capture(self) -> bool:
"""Whether this expert supports routing replay capture.
Subclasses backed by a kernel that exposes routed expert IDs
(e.g. FlashInfer's ``routing_replay_out``) should override.
"""
return False
def set_capture_fn(
self,
capture_fn: Callable[[torch.Tensor], None] | None,
) -> None:
self.routing_replay_capture_fn = capture_fn
if capture_fn is None:
self._routing_replay_buffer = None
return
self._routing_replay_buffer = torch.empty(
(self.moe_config.max_num_tokens, self.moe_config.experts_per_token),
dtype=torch.int16,
device=self.moe_config.device,
)
def _maybe_make_routing_replay_buffer(
self,
num_tokens: int,
device: torch.device,
) -> torch.Tensor | None:
if self.routing_replay_capture_fn is None:
return None
buf = self._routing_replay_buffer
assert buf is not None
if buf.shape[0] < num_tokens or buf.device != device:
raise ValueError(
"Routing replay buffer was initialized for "
f"{buf.shape[0]} tokens on {buf.device}, but the kernel "
f"received {num_tokens} tokens on {device}."
)
return buf
def _maybe_dispatch_routing_replay(
self,
routing_replay_out: torch.Tensor | None,
num_tokens: int,
) -> None:
if routing_replay_out is None or self.routing_replay_capture_fn is None:
return
self.routing_replay_capture_fn(routing_replay_out[:num_tokens])
def apply(
self,
hidden_states: torch.Tensor,
@@ -90,10 +90,20 @@ class RoutedExpertsCapturer:
) -> None:
hf_config = vllm_config.model_config.hf_text_config
num_experts_per_tok = _get_num_experts_per_tok(hf_config)
num_layers = hf_config.num_hidden_layers
logger.info(
"RoutedExpertsCapturer: allocating buffer with "
"max_tokens=%d, num_layers=%d, num_experts_per_tok=%d "
"(hf_config.model_type=%s)",
max_num_batched_tokens,
num_layers,
num_experts_per_tok,
getattr(hf_config, "model_type", "unknown"),
)
self.device_buffer = torch.zeros(
(
max_num_batched_tokens,
hf_config.num_hidden_layers,
num_layers,
num_experts_per_tok,
),
# Use int32 for the device / host transit buffers: it
@@ -190,6 +190,7 @@ def flashinfer_trtllm_mxint4_moe(
topk_group: int | None = None,
e_score_correction_bias: torch.Tensor | None = None,
routing_method_type: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor:
"""
Apply FlashInfer TensorRT-LLM MxInt4 MoE kernel.
@@ -262,6 +263,7 @@ def flashinfer_trtllm_mxint4_moe(
do_finalize=True,
output=None,
tune_max_num_tokens=8192,
routing_replay_out=routing_replay_out,
)
if isinstance(out, (tuple, list)):
out = out[0]