[Refactor] Replace activation: str with MoEActivation enum (#33843)
Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
@@ -9,6 +9,7 @@ import torch
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
@@ -64,9 +65,9 @@ def _supports_quant_scheme(
|
||||
return (weight_key, activation_key) in SUPPORTED_W_A
|
||||
|
||||
|
||||
def _supports_activation(activation: str) -> bool:
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
"""Supports silu activation only."""
|
||||
return activation in ["silu"]
|
||||
return activation in [MoEActivation.SILU]
|
||||
|
||||
|
||||
def _supports_routing_method(
|
||||
@@ -267,7 +268,7 @@ def flashinfer_trtllm_fp4_moe(
|
||||
x: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
router_logits: torch.Tensor,
|
||||
top_k: int,
|
||||
activation: str,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
num_expert_group: int | None,
|
||||
topk_group: int | None,
|
||||
@@ -297,7 +298,7 @@ def flashinfer_trtllm_fp4_moe(
|
||||
from vllm.model_executor.models.llama4 import Llama4MoE
|
||||
|
||||
# https://github.com/flashinfer-ai/flashinfer/blob/f0277fd1bff90e309e5c19cab36c5dae056d685d/flashinfer/fused_moe/core.py#L2404
|
||||
assert activation == "silu", (
|
||||
assert activation == MoEActivation.SILU, (
|
||||
"Only SiLU activation is supported for FlashInfer TRTLLM FP4 MoE. "
|
||||
f"{activation} found instead."
|
||||
)
|
||||
@@ -365,7 +366,7 @@ def flashinfer_trtllm_fp4_routed_moe(
|
||||
topk_ids: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
top_k: int,
|
||||
activation: str,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
@@ -387,7 +388,7 @@ def flashinfer_trtllm_fp4_routed_moe(
|
||||
import flashinfer
|
||||
|
||||
# https://github.com/flashinfer-ai/flashinfer/blob/f0277fd1bff90e309e5c19cab36c5dae056d685d/flashinfer/fused_moe/core.py#L2535
|
||||
assert activation == "silu", (
|
||||
assert activation == MoEActivation.SILU, (
|
||||
"Only SiLU activation is supported for FlashInfer TRTLLM FP4 Routed MoE. "
|
||||
f"{activation} found instead."
|
||||
)
|
||||
|
||||
@@ -6,6 +6,7 @@ from typing import Any
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import triton
|
||||
from vllm.utils.import_utils import has_triton_kernels
|
||||
@@ -88,7 +89,7 @@ def _can_support_mxfp4(
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
apply_router_weight_on_input: bool = False,
|
||||
scoring_func: str = "softmax",
|
||||
activation: str = "swigluoai",
|
||||
activation: MoEActivation = MoEActivation.SWIGLUOAI,
|
||||
expert_load_view: torch.Tensor | None = None,
|
||||
logical_to_physical_map: torch.Tensor | None = None,
|
||||
logical_replica_count: torch.Tensor | None = None,
|
||||
@@ -101,7 +102,7 @@ def _can_support_mxfp4(
|
||||
or e_score_correction_bias
|
||||
or apply_router_weight_on_input
|
||||
or scoring_func != "softmax"
|
||||
or activation != "swigluoai"
|
||||
or activation != MoEActivation.SWIGLUOAI
|
||||
or expert_load_view
|
||||
or logical_to_physical_map
|
||||
or logical_replica_count
|
||||
|
||||
Reference in New Issue
Block a user