[RL Infra][FlashInfer] Enable router replay output from FlashInfer monolithic MoE kernel (#44214)
Signed-off-by: Xuanyu Zhang <xuanyu.zhang@mistral.ai> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Claude
aoshen02
OpenAI Codex
parent
58b2012aa2
commit
97a668152b
@@ -34,6 +34,9 @@ class TrtLlmBf16ExpertsBase:
|
||||
monolithic interfaces.
|
||||
"""
|
||||
|
||||
def supports_routing_replay_capture(self) -> bool:
|
||||
return True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
@@ -246,7 +249,11 @@ class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonol
|
||||
|
||||
assert activation in [MoEActivation.SILU, MoEActivation.RELU2_NO_MUL]
|
||||
|
||||
return flashinfer.fused_moe.trtllm_bf16_moe(
|
||||
routing_replay_out = self._maybe_make_routing_replay_buffer(
|
||||
num_tokens=hidden_states.shape[0],
|
||||
device=hidden_states.device,
|
||||
)
|
||||
out = flashinfer.fused_moe.trtllm_bf16_moe(
|
||||
routing_logits=router_logits,
|
||||
routing_bias=e_score_correction_bias,
|
||||
hidden_states=hidden_states,
|
||||
@@ -263,4 +270,9 @@ class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonol
|
||||
routing_method_type=self.routing_method_type,
|
||||
activation_type=activation_to_flashinfer_int(activation),
|
||||
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
|
||||
routing_replay_out=routing_replay_out,
|
||||
)
|
||||
self._maybe_dispatch_routing_replay(
|
||||
routing_replay_out, num_tokens=hidden_states.shape[0]
|
||||
)
|
||||
return out
|
||||
|
||||
@@ -265,6 +265,9 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
Fp8 TRTLLM-Gen MoE kernels. Supports monolithic interface.
|
||||
"""
|
||||
|
||||
def supports_routing_replay_capture(self) -> bool:
|
||||
return True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
@@ -403,6 +406,11 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
n_group = num_expert_group or 0
|
||||
selected_topk_group = topk_group or 0
|
||||
|
||||
routing_replay_out = self._maybe_make_routing_replay_buffer(
|
||||
num_tokens=hidden_states.shape[0],
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
kwargs = dict(
|
||||
routing_logits=router_logits,
|
||||
routing_bias=e_score_correction_bias,
|
||||
@@ -427,11 +435,16 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
use_shuffled_weight=use_shuffled_weight,
|
||||
weight_layout=weight_layout,
|
||||
fp8_quantization_type=fp8_quant_type,
|
||||
routing_replay_out=routing_replay_out,
|
||||
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
|
||||
)
|
||||
if is_mxfp8 or activation == MoEActivation.RELU2_NO_MUL:
|
||||
kwargs["activation_type"] = activation_type
|
||||
return flashinfer.fused_moe.trtllm_fp8_block_scale_moe(**kwargs)
|
||||
result = flashinfer.fused_moe.trtllm_fp8_block_scale_moe(**kwargs)
|
||||
self._maybe_dispatch_routing_replay(
|
||||
routing_replay_out, num_tokens=hidden_states.shape[0]
|
||||
)
|
||||
return result
|
||||
|
||||
def _apply_per_tensor(
|
||||
self,
|
||||
@@ -464,6 +477,11 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
else:
|
||||
assert not apply_router_weight_on_input
|
||||
|
||||
routing_replay_out = self._maybe_make_routing_replay_buffer(
|
||||
num_tokens=hidden_states.shape[0],
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
out = flashinfer.fused_moe.trtllm_fp8_per_tensor_scale_moe(
|
||||
routing_logits=router_logits,
|
||||
routing_bias=e_score_correction_bias,
|
||||
@@ -485,6 +503,10 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
routing_method_type=self.routing_method_type,
|
||||
activation_type=activation_type,
|
||||
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
|
||||
routing_replay_out=routing_replay_out,
|
||||
)
|
||||
self._maybe_dispatch_routing_replay(
|
||||
routing_replay_out, num_tokens=hidden_states.shape[0]
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@@ -122,6 +122,9 @@ class TrtLlmMxfp4ExpertsMonolithic(
|
||||
Wraps flashinfer.trtllm_fp4_block_scale_moe().
|
||||
"""
|
||||
|
||||
def supports_routing_replay_capture(self) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(
|
||||
moe_parallel_config: FusedMoEParallelConfig,
|
||||
@@ -184,6 +187,10 @@ class TrtLlmMxfp4ExpertsMonolithic(
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
routing_replay_out = self._maybe_make_routing_replay_buffer(
|
||||
num_tokens=hidden_states.shape[0],
|
||||
device=hidden_states.device,
|
||||
)
|
||||
trtllm_fp4_block_scale_moe(
|
||||
routing_logits=router_logits.to(torch.bfloat16),
|
||||
routing_bias=None,
|
||||
@@ -213,8 +220,11 @@ class TrtLlmMxfp4ExpertsMonolithic(
|
||||
do_finalize=True,
|
||||
tune_max_num_tokens=max(self.max_capture_size, 1),
|
||||
output=output,
|
||||
routing_replay_out=routing_replay_out,
|
||||
)
|
||||
self._maybe_dispatch_routing_replay(
|
||||
routing_replay_out, num_tokens=hidden_states.shape[0]
|
||||
)
|
||||
|
||||
return output
|
||||
|
||||
|
||||
|
||||
@@ -122,6 +122,9 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
# The kernel handles quantization internally.
|
||||
return True
|
||||
|
||||
def supports_routing_replay_capture(self) -> bool:
|
||||
return True
|
||||
|
||||
def apply(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -144,7 +147,12 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
|
||||
assert self.w1_scale is not None
|
||||
assert self.w2_scale is not None
|
||||
return flashinfer_trtllm_mxint4_moe(
|
||||
|
||||
routing_replay_out = self._maybe_make_routing_replay_buffer(
|
||||
num_tokens=hidden_states.shape[0],
|
||||
device=hidden_states.device,
|
||||
)
|
||||
result = flashinfer_trtllm_mxint4_moe(
|
||||
x=hidden_states,
|
||||
router_logits=router_logits,
|
||||
w13_weight_packed=w1,
|
||||
@@ -160,4 +168,9 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
topk_group=topk_group,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
routing_method_type=self.routing_method,
|
||||
routing_replay_out=routing_replay_out,
|
||||
)
|
||||
self._maybe_dispatch_routing_replay(
|
||||
routing_replay_out, num_tokens=hidden_states.shape[0]
|
||||
)
|
||||
return result
|
||||
|
||||
@@ -435,6 +435,9 @@ class TrtLlmNvFp4ExpertsMonolithic(
|
||||
Monolithic version of the kernel (router + experts).
|
||||
"""
|
||||
|
||||
def supports_routing_replay_capture(self) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
|
||||
"""The modular implementation should be used for the Dp/Ep or EPLB case."""
|
||||
@@ -509,10 +512,14 @@ class TrtLlmNvFp4ExpertsMonolithic(
|
||||
|
||||
output1_scale_gate_scalar = self.quant_config.g1_alphas
|
||||
|
||||
routing_replay_out = self._maybe_make_routing_replay_buffer(
|
||||
num_tokens=hidden_states.shape[0],
|
||||
device=hidden_states.device,
|
||||
)
|
||||
# Invoke kernel.
|
||||
# NOTE: Activation padding and output
|
||||
# truncation are handled by the MoE runner's
|
||||
return flashinfer.fused_moe.trtllm_fp4_block_scale_moe(
|
||||
result = flashinfer.fused_moe.trtllm_fp4_block_scale_moe(
|
||||
routing_logits=router_logits,
|
||||
routing_bias=e_score_correction_bias,
|
||||
hidden_states=hidden_states,
|
||||
@@ -544,4 +551,9 @@ class TrtLlmNvFp4ExpertsMonolithic(
|
||||
activation_type=activation_to_flashinfer_int(activation),
|
||||
per_token_scale=per_token_scale,
|
||||
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
|
||||
routing_replay_out=routing_replay_out,
|
||||
)[0]
|
||||
self._maybe_dispatch_routing_replay(
|
||||
routing_replay_out, num_tokens=hidden_states.shape[0]
|
||||
)
|
||||
return result
|
||||
|
||||
@@ -1000,6 +1000,57 @@ class FusedMoEExpertsMonolithic(FusedMoEExperts):
|
||||
def is_monolithic() -> bool:
|
||||
return True
|
||||
|
||||
routing_replay_capture_fn: Callable[[torch.Tensor], None] | None = None
|
||||
_routing_replay_buffer: torch.Tensor | None = None
|
||||
|
||||
def supports_routing_replay_capture(self) -> bool:
|
||||
"""Whether this expert supports routing replay capture.
|
||||
|
||||
Subclasses backed by a kernel that exposes routed expert IDs
|
||||
(e.g. FlashInfer's ``routing_replay_out``) should override.
|
||||
"""
|
||||
return False
|
||||
|
||||
def set_capture_fn(
|
||||
self,
|
||||
capture_fn: Callable[[torch.Tensor], None] | None,
|
||||
) -> None:
|
||||
self.routing_replay_capture_fn = capture_fn
|
||||
if capture_fn is None:
|
||||
self._routing_replay_buffer = None
|
||||
return
|
||||
self._routing_replay_buffer = torch.empty(
|
||||
(self.moe_config.max_num_tokens, self.moe_config.experts_per_token),
|
||||
dtype=torch.int16,
|
||||
device=self.moe_config.device,
|
||||
)
|
||||
|
||||
def _maybe_make_routing_replay_buffer(
|
||||
self,
|
||||
num_tokens: int,
|
||||
device: torch.device,
|
||||
) -> torch.Tensor | None:
|
||||
if self.routing_replay_capture_fn is None:
|
||||
return None
|
||||
buf = self._routing_replay_buffer
|
||||
assert buf is not None
|
||||
if buf.shape[0] < num_tokens or buf.device != device:
|
||||
raise ValueError(
|
||||
"Routing replay buffer was initialized for "
|
||||
f"{buf.shape[0]} tokens on {buf.device}, but the kernel "
|
||||
f"received {num_tokens} tokens on {device}."
|
||||
)
|
||||
return buf
|
||||
|
||||
def _maybe_dispatch_routing_replay(
|
||||
self,
|
||||
routing_replay_out: torch.Tensor | None,
|
||||
num_tokens: int,
|
||||
) -> None:
|
||||
if routing_replay_out is None or self.routing_replay_capture_fn is None:
|
||||
return
|
||||
self.routing_replay_capture_fn(routing_replay_out[:num_tokens])
|
||||
|
||||
def apply(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
|
||||
@@ -90,10 +90,20 @@ class RoutedExpertsCapturer:
|
||||
) -> None:
|
||||
hf_config = vllm_config.model_config.hf_text_config
|
||||
num_experts_per_tok = _get_num_experts_per_tok(hf_config)
|
||||
num_layers = hf_config.num_hidden_layers
|
||||
logger.info(
|
||||
"RoutedExpertsCapturer: allocating buffer with "
|
||||
"max_tokens=%d, num_layers=%d, num_experts_per_tok=%d "
|
||||
"(hf_config.model_type=%s)",
|
||||
max_num_batched_tokens,
|
||||
num_layers,
|
||||
num_experts_per_tok,
|
||||
getattr(hf_config, "model_type", "unknown"),
|
||||
)
|
||||
self.device_buffer = torch.zeros(
|
||||
(
|
||||
max_num_batched_tokens,
|
||||
hf_config.num_hidden_layers,
|
||||
num_layers,
|
||||
num_experts_per_tok,
|
||||
),
|
||||
# Use int32 for the device / host transit buffers: it
|
||||
|
||||
@@ -190,6 +190,7 @@ def flashinfer_trtllm_mxint4_moe(
|
||||
topk_group: int | None = None,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routing_method_type: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Apply FlashInfer TensorRT-LLM MxInt4 MoE kernel.
|
||||
@@ -262,6 +263,7 @@ def flashinfer_trtllm_mxint4_moe(
|
||||
do_finalize=True,
|
||||
output=None,
|
||||
tune_max_num_tokens=8192,
|
||||
routing_replay_out=routing_replay_out,
|
||||
)
|
||||
if isinstance(out, (tuple, list)):
|
||||
out = out[0]
|
||||
|
||||
Reference in New Issue
Block a user