[Bugfix][MoE] Tune FlashInfer experts to scheduler token limit (#52989)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
4f6885fffc
commit
bfb6c13499
@@ -72,6 +72,7 @@ def test_flashinfer_swigluoai_params_are_forwarded(activation, monkeypatch):
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
in_dtype=torch.bfloat16,
|
||||
routing_method=RoutingMethodType.TopK,
|
||||
max_num_tokens=16_384,
|
||||
)
|
||||
quant_config = FusedMoEQuantConfig.make(
|
||||
gemm1_alpha=1.702,
|
||||
@@ -110,6 +111,7 @@ def test_flashinfer_swigluoai_params_are_forwarded(activation, monkeypatch):
|
||||
|
||||
assert experts._supports_activation(activation)
|
||||
assert call_args["activation_type"] == ActivationType.Swiglu
|
||||
assert call_args["tune_max_num_tokens"] == 16_384
|
||||
for name, value in (
|
||||
("swiglu_alpha", 1.702),
|
||||
("swiglu_beta", 1.0),
|
||||
|
||||
@@ -13,6 +13,7 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceNoOP,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import fi_moe_largest_bucket
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
activation_to_flashinfer_type,
|
||||
)
|
||||
@@ -388,6 +389,7 @@ class FlashInferExperts(mk.FusedMoEExpertsModular):
|
||||
use_deepseek_fp8_block_scale=self.use_deepseek_fp8_block_scale,
|
||||
use_mxfp8_act_scaling=use_mxfp8_act_scaling,
|
||||
use_w4_group_scaling=use_w4_group_scaling,
|
||||
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
|
||||
)
|
||||
|
||||
def moe_sum(self, input: torch.Tensor, output: torch.Tensor) -> None:
|
||||
|
||||
@@ -219,6 +219,7 @@ class TrtLlmBf16ExpertsModular(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsModular)
|
||||
weight_layout=WeightLayout.BlockMajorK,
|
||||
do_finalize=True,
|
||||
activation_type=activation_to_flashinfer_int(activation),
|
||||
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
|
||||
)
|
||||
# FlashInfer's BF16 routed wrapper does not expose an output= argument.
|
||||
output.copy_(result[0] if isinstance(result, list) else result)
|
||||
|
||||
@@ -39,6 +39,7 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceNoOP,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
fi_moe_largest_bucket,
|
||||
trtllm_moe_pack_topk_ids_weights,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
@@ -549,6 +550,7 @@ class TrtLlmBf16LoRAExperts(_TrtLlmLoRAExpertsBase):
|
||||
routing_method_type=self.routing_method_type,
|
||||
do_finalize=do_finalize,
|
||||
output=output if do_finalize else None,
|
||||
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
|
||||
)
|
||||
if not do_finalize:
|
||||
# [gemm2_output, expert_weights, expanded_idx, gemm1_activation]
|
||||
|
||||
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import fi_moe_largest_bucket
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kInt4Static32,
|
||||
@@ -169,6 +170,7 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
routing_method_type=self.routing_method,
|
||||
routing_replay_out=routing_replay_out,
|
||||
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
|
||||
)
|
||||
self._maybe_dispatch_routing_replay(
|
||||
routing_replay_out, num_tokens=hidden_states.shape[0]
|
||||
|
||||
@@ -191,6 +191,7 @@ def flashinfer_trtllm_mxint4_moe(
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routing_method_type: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
tune_max_num_tokens: int = 8192,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Apply FlashInfer TensorRT-LLM MxInt4 MoE kernel.
|
||||
@@ -211,6 +212,7 @@ def flashinfer_trtllm_mxint4_moe(
|
||||
topk_group: Top-k within groups (default: None -> 0)
|
||||
e_score_correction_bias: Optional routing bias. dtype: bfloat16
|
||||
routing_method_type: FlashInfer RoutingMethodType enum value
|
||||
tune_max_num_tokens: Maximum token count covered by autotuning.
|
||||
|
||||
Returns:
|
||||
Output tensor from MoE layer. dtype: same as x (bfloat16)
|
||||
@@ -262,7 +264,7 @@ def flashinfer_trtllm_mxint4_moe(
|
||||
enable_pdl=None,
|
||||
do_finalize=True,
|
||||
output=None,
|
||||
tune_max_num_tokens=8192,
|
||||
tune_max_num_tokens=tune_max_num_tokens,
|
||||
routing_replay_out=routing_replay_out,
|
||||
)
|
||||
if isinstance(out, (tuple, list)):
|
||||
|
||||
Reference in New Issue
Block a user