[Bugfix][MoE] Tune FlashInfer experts to scheduler token limit (#52989)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
4f6885fffc
commit
bfb6c13499
@@ -191,6 +191,7 @@ def flashinfer_trtllm_mxint4_moe(
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routing_method_type: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
tune_max_num_tokens: int = 8192,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Apply FlashInfer TensorRT-LLM MxInt4 MoE kernel.
|
||||
@@ -211,6 +212,7 @@ def flashinfer_trtllm_mxint4_moe(
|
||||
topk_group: Top-k within groups (default: None -> 0)
|
||||
e_score_correction_bias: Optional routing bias. dtype: bfloat16
|
||||
routing_method_type: FlashInfer RoutingMethodType enum value
|
||||
tune_max_num_tokens: Maximum token count covered by autotuning.
|
||||
|
||||
Returns:
|
||||
Output tensor from MoE layer. dtype: same as x (bfloat16)
|
||||
@@ -262,7 +264,7 @@ def flashinfer_trtllm_mxint4_moe(
|
||||
enable_pdl=None,
|
||||
do_finalize=True,
|
||||
output=None,
|
||||
tune_max_num_tokens=8192,
|
||||
tune_max_num_tokens=tune_max_num_tokens,
|
||||
routing_replay_out=routing_replay_out,
|
||||
)
|
||||
if isinstance(out, (tuple, list)):
|
||||
|
||||
Reference in New Issue
Block a user