Co-authored-by: vllm-ci-failure-analyzer <kevin@inferact.ai>
This commit is contained in:
co-authored by
vllm-ci-failure-analyzer
parent
7a2fdbaac4
commit
ba53da60bb
@@ -72,7 +72,6 @@ def test_flashinfer_swigluoai_params_are_forwarded(activation, monkeypatch):
|
|||||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||||
in_dtype=torch.bfloat16,
|
in_dtype=torch.bfloat16,
|
||||||
routing_method=RoutingMethodType.TopK,
|
routing_method=RoutingMethodType.TopK,
|
||||||
max_num_tokens=16_384,
|
|
||||||
)
|
)
|
||||||
quant_config = FusedMoEQuantConfig.make(
|
quant_config = FusedMoEQuantConfig.make(
|
||||||
gemm1_alpha=1.702,
|
gemm1_alpha=1.702,
|
||||||
@@ -111,7 +110,6 @@ def test_flashinfer_swigluoai_params_are_forwarded(activation, monkeypatch):
|
|||||||
|
|
||||||
assert experts._supports_activation(activation)
|
assert experts._supports_activation(activation)
|
||||||
assert call_args["activation_type"] == ActivationType.Swiglu
|
assert call_args["activation_type"] == ActivationType.Swiglu
|
||||||
assert call_args["tune_max_num_tokens"] == 16_384
|
|
||||||
for name, value in (
|
for name, value in (
|
||||||
("swiglu_alpha", 1.702),
|
("swiglu_alpha", 1.702),
|
||||||
("swiglu_beta", 1.0),
|
("swiglu_beta", 1.0),
|
||||||
|
|||||||
@@ -13,7 +13,6 @@ from vllm.model_executor.layers.fused_moe.config import (
|
|||||||
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||||
TopKWeightAndReduceNoOP,
|
TopKWeightAndReduceNoOP,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.layers.fused_moe.utils import fi_moe_largest_bucket
|
|
||||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||||
activation_to_flashinfer_type,
|
activation_to_flashinfer_type,
|
||||||
)
|
)
|
||||||
@@ -389,7 +388,6 @@ class FlashInferExperts(mk.FusedMoEExpertsModular):
|
|||||||
use_deepseek_fp8_block_scale=self.use_deepseek_fp8_block_scale,
|
use_deepseek_fp8_block_scale=self.use_deepseek_fp8_block_scale,
|
||||||
use_mxfp8_act_scaling=use_mxfp8_act_scaling,
|
use_mxfp8_act_scaling=use_mxfp8_act_scaling,
|
||||||
use_w4_group_scaling=use_w4_group_scaling,
|
use_w4_group_scaling=use_w4_group_scaling,
|
||||||
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
def moe_sum(self, input: torch.Tensor, output: torch.Tensor) -> None:
|
def moe_sum(self, input: torch.Tensor, output: torch.Tensor) -> None:
|
||||||
|
|||||||
@@ -220,7 +220,6 @@ class TrtLlmBf16ExpertsModular(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsModular)
|
|||||||
weight_layout=WeightLayout.BlockMajorK,
|
weight_layout=WeightLayout.BlockMajorK,
|
||||||
do_finalize=True,
|
do_finalize=True,
|
||||||
activation_type=activation_to_flashinfer_int(activation),
|
activation_type=activation_to_flashinfer_int(activation),
|
||||||
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
|
|
||||||
)
|
)
|
||||||
# FlashInfer's BF16 routed wrapper does not expose an output= argument.
|
# FlashInfer's BF16 routed wrapper does not expose an output= argument.
|
||||||
output.copy_(result[0] if isinstance(result, list) else result)
|
output.copy_(result[0] if isinstance(result, list) else result)
|
||||||
|
|||||||
@@ -39,7 +39,6 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
|||||||
TopKWeightAndReduceNoOP,
|
TopKWeightAndReduceNoOP,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.layers.fused_moe.utils import (
|
from vllm.model_executor.layers.fused_moe.utils import (
|
||||||
fi_moe_largest_bucket,
|
|
||||||
trtllm_moe_pack_topk_ids_weights,
|
trtllm_moe_pack_topk_ids_weights,
|
||||||
)
|
)
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
@@ -550,7 +549,6 @@ class TrtLlmBf16LoRAExperts(_TrtLlmLoRAExpertsBase):
|
|||||||
routing_method_type=self.routing_method_type,
|
routing_method_type=self.routing_method_type,
|
||||||
do_finalize=do_finalize,
|
do_finalize=do_finalize,
|
||||||
output=output if do_finalize else None,
|
output=output if do_finalize else None,
|
||||||
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
|
|
||||||
)
|
)
|
||||||
if not do_finalize:
|
if not do_finalize:
|
||||||
# [gemm2_output, expert_weights, expanded_idx, gemm1_activation]
|
# [gemm2_output, expert_weights, expanded_idx, gemm1_activation]
|
||||||
|
|||||||
@@ -11,7 +11,6 @@ from vllm.model_executor.layers.fused_moe.config import (
|
|||||||
FusedMoEQuantConfig,
|
FusedMoEQuantConfig,
|
||||||
RoutingMethodType,
|
RoutingMethodType,
|
||||||
)
|
)
|
||||||
from vllm.model_executor.layers.fused_moe.utils import fi_moe_largest_bucket
|
|
||||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||||
QuantKey,
|
QuantKey,
|
||||||
kInt4Static32,
|
kInt4Static32,
|
||||||
@@ -170,7 +169,6 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
|||||||
e_score_correction_bias=e_score_correction_bias,
|
e_score_correction_bias=e_score_correction_bias,
|
||||||
routing_method_type=self.routing_method,
|
routing_method_type=self.routing_method,
|
||||||
routing_replay_out=routing_replay_out,
|
routing_replay_out=routing_replay_out,
|
||||||
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
|
|
||||||
)
|
)
|
||||||
self._maybe_dispatch_routing_replay(
|
self._maybe_dispatch_routing_replay(
|
||||||
routing_replay_out, num_tokens=hidden_states.shape[0]
|
routing_replay_out, num_tokens=hidden_states.shape[0]
|
||||||
|
|||||||
@@ -191,7 +191,6 @@ def flashinfer_trtllm_mxint4_moe(
|
|||||||
e_score_correction_bias: torch.Tensor | None = None,
|
e_score_correction_bias: torch.Tensor | None = None,
|
||||||
routing_method_type: int | None = None,
|
routing_method_type: int | None = None,
|
||||||
routing_replay_out: torch.Tensor | None = None,
|
routing_replay_out: torch.Tensor | None = None,
|
||||||
tune_max_num_tokens: int = 8192,
|
|
||||||
) -> torch.Tensor:
|
) -> torch.Tensor:
|
||||||
"""
|
"""
|
||||||
Apply FlashInfer TensorRT-LLM MxInt4 MoE kernel.
|
Apply FlashInfer TensorRT-LLM MxInt4 MoE kernel.
|
||||||
@@ -212,7 +211,6 @@ def flashinfer_trtllm_mxint4_moe(
|
|||||||
topk_group: Top-k within groups (default: None -> 0)
|
topk_group: Top-k within groups (default: None -> 0)
|
||||||
e_score_correction_bias: Optional routing bias. dtype: bfloat16
|
e_score_correction_bias: Optional routing bias. dtype: bfloat16
|
||||||
routing_method_type: FlashInfer RoutingMethodType enum value
|
routing_method_type: FlashInfer RoutingMethodType enum value
|
||||||
tune_max_num_tokens: Maximum token count covered by autotuning.
|
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Output tensor from MoE layer. dtype: same as x (bfloat16)
|
Output tensor from MoE layer. dtype: same as x (bfloat16)
|
||||||
@@ -264,7 +262,7 @@ def flashinfer_trtllm_mxint4_moe(
|
|||||||
enable_pdl=None,
|
enable_pdl=None,
|
||||||
do_finalize=True,
|
do_finalize=True,
|
||||||
output=None,
|
output=None,
|
||||||
tune_max_num_tokens=tune_max_num_tokens,
|
tune_max_num_tokens=8192,
|
||||||
routing_replay_out=routing_replay_out,
|
routing_replay_out=routing_replay_out,
|
||||||
)
|
)
|
||||||
if isinstance(out, (tuple, list)):
|
if isinstance(out, (tuple, list)):
|
||||||
|
|||||||
Reference in New Issue
Block a user