[MoE Refactor] FusedMoE/MoERunner inversion refactor (#41184)
Signed-off-by: Bill Nell <bnell@redhat.com>
This commit is contained in:
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from contextlib import contextmanager
|
||||
from typing import Any, TypeAlias
|
||||
from typing import Any
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.activation import (
|
||||
MoEActivation,
|
||||
@@ -20,7 +20,6 @@ from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.layer import (
|
||||
FusedMoE,
|
||||
FusedMoeWeightScaleSupported,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import (
|
||||
@@ -28,10 +27,17 @@ from vllm.model_executor.layers.fused_moe.modular_kernel import (
|
||||
FusedMoEExpertsModular,
|
||||
FusedMoEPrepareAndFinalizeModular,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts import (
|
||||
FusedMoeWeightScaleSupported,
|
||||
RoutedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.fused_moe_router import (
|
||||
FusedMoERouter,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
|
||||
from vllm.model_executor.layers.fused_moe.runner.moe_runner import (
|
||||
MoERunner,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.runner.shared_experts import (
|
||||
SharedExperts,
|
||||
)
|
||||
@@ -43,10 +49,6 @@ from vllm.triton_utils import HAS_TRITON
|
||||
_config: dict[str, Any] | None = None
|
||||
|
||||
|
||||
# Temporary alias for FusedMoE, eventually we be its own class.
|
||||
RoutedExperts: TypeAlias = FusedMoE
|
||||
|
||||
|
||||
@contextmanager
|
||||
def override_config(config):
|
||||
global _config
|
||||
@@ -74,6 +76,7 @@ __all__ = [
|
||||
"FusedMoEActivationFormat",
|
||||
"FusedMoEPrepareAndFinalizeModular",
|
||||
"GateLinear",
|
||||
"MoERunner",
|
||||
"RoutingMethodType",
|
||||
"RoutedExperts",
|
||||
"SharedExperts",
|
||||
|
||||
@@ -1236,7 +1236,7 @@ class FusedMoEConfig:
|
||||
num_experts: int
|
||||
experts_per_token: int
|
||||
hidden_dim: int
|
||||
intermediate_size_per_partition: int
|
||||
intermediate_size: int
|
||||
num_local_experts: int
|
||||
num_logical_experts: int
|
||||
activation: MoEActivation
|
||||
@@ -1258,7 +1258,6 @@ class FusedMoEConfig:
|
||||
moe_backend: MoEBackend = "auto"
|
||||
max_num_tokens: int = SchedulerConfig.DEFAULT_MAX_NUM_BATCHED_TOKENS_FOR_BATCHED_DP
|
||||
has_bias: bool = False
|
||||
is_act_and_mul: bool = True
|
||||
is_lora_enabled: bool = False
|
||||
|
||||
# SwiGLU clamp limit. When set, backends that do not implement the clamp
|
||||
@@ -1268,7 +1267,18 @@ class FusedMoEConfig:
|
||||
|
||||
max_capture_size: int = 0
|
||||
|
||||
# Set by __post_init__
|
||||
intermediate_size_per_partition: int = -1
|
||||
rocm_aiter_fmoe_enabled: bool = False
|
||||
aiter_fmoe_shared_expert_enabled: bool = False
|
||||
|
||||
def __post_init__(self):
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
tp_size = self.moe_parallel_config.tp_size
|
||||
assert self.intermediate_size % tp_size == 0
|
||||
self.intermediate_size_per_partition = self.intermediate_size // tp_size
|
||||
|
||||
if self.dp_size > 1:
|
||||
logger.debug_once(
|
||||
"Using FusedMoEConfig::max_num_tokens=%d", self.max_num_tokens
|
||||
@@ -1286,6 +1296,32 @@ class FusedMoEConfig:
|
||||
self.intermediate_size_per_partition
|
||||
)
|
||||
|
||||
if self.is_act_and_mul:
|
||||
self.rocm_aiter_fmoe_enabled = rocm_aiter_ops.is_fused_moe_enabled()
|
||||
self.aiter_fmoe_shared_expert_enabled = (
|
||||
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
|
||||
)
|
||||
|
||||
if self.use_mori_kernels:
|
||||
assert self.rocm_aiter_fmoe_enabled, (
|
||||
"Mori needs to be used with aiter fused_moe for now."
|
||||
)
|
||||
assert not self.aiter_fmoe_shared_expert_enabled, (
|
||||
"Mori does not support fusion shared expert now. "
|
||||
"Turn it off by setting VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0"
|
||||
)
|
||||
|
||||
if not self.is_act_and_mul and not (
|
||||
current_platform.is_cuda_alike() or current_platform.is_xpu()
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"is_act_and_mul=False is supported only for CUDA, XPU and ROCm for now"
|
||||
)
|
||||
|
||||
@property
|
||||
def is_act_and_mul(self) -> bool:
|
||||
return self.activation.is_gated
|
||||
|
||||
@property
|
||||
def tp_size(self):
|
||||
return self.moe_parallel_config.tp_size
|
||||
|
||||
@@ -21,7 +21,7 @@ from vllm.model_executor.layers.fused_moe.modular_kernel import (
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
|
||||
from vllm.model_executor.layers.fused_moe.runner.moe_runner import MoERunner
|
||||
|
||||
|
||||
def _make_eep_experts(
|
||||
@@ -58,10 +58,10 @@ def _make_eep_experts(
|
||||
|
||||
|
||||
def make_eep_staged_quant_method(
|
||||
module: "FusedMoE",
|
||||
module: "MoERunner",
|
||||
moe_config: FusedMoEConfig,
|
||||
) -> FusedMoEMethodBase | None:
|
||||
quant_method = module.quant_method
|
||||
quant_method = module._quant_method
|
||||
if not quant_method.supports_internal_mk:
|
||||
return None
|
||||
if getattr(quant_method, "wraps_legacy_quant_method", False):
|
||||
|
||||
@@ -53,7 +53,7 @@ class ActivationMethod(IntEnum):
|
||||
GELU = 1
|
||||
|
||||
|
||||
aiter_topK_meta_data = None
|
||||
aiter_topK_meta_data: tuple[torch.Tensor, torch.Tensor] | None = None
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
|
||||
@@ -22,6 +22,7 @@ from vllm.model_executor.layers.quantization.base_config import (
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
|
||||
from vllm.model_executor.layers.fused_moe.runner.shared_experts import SharedExperts
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -51,7 +52,7 @@ class FusedMoEMethodBase(QuantizeMethodBase):
|
||||
@abstractmethod
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
layer: "RoutedExperts",
|
||||
num_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
@@ -117,7 +118,7 @@ class FusedMoEMethodBase(QuantizeMethodBase):
|
||||
def select_gemm_impl(
|
||||
self,
|
||||
prepare_finalize: FusedMoEPrepareAndFinalizeModular,
|
||||
layer: torch.nn.Module,
|
||||
layer: "RoutedExperts",
|
||||
) -> FusedMoEExpertsModular:
|
||||
# based on the all2all implementation, select the appropriate
|
||||
# gemm implementation
|
||||
@@ -128,7 +129,7 @@ class FusedMoEMethodBase(QuantizeMethodBase):
|
||||
|
||||
@abstractmethod
|
||||
def get_fused_moe_quant_config(
|
||||
self, layer: torch.nn.Module
|
||||
self, layer: "RoutedExperts"
|
||||
) -> FusedMoEQuantConfig | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -143,6 +144,14 @@ class FusedMoEMethodBase(QuantizeMethodBase):
|
||||
"""Whether to skip the padding in the forward before applying the moe method."""
|
||||
return False
|
||||
|
||||
@property
|
||||
def has_unpadded_output(self) -> bool:
|
||||
"""
|
||||
Indicates that the hidden_states output might be the unpadded
|
||||
hidden_states shape rather than the full padded shape.
|
||||
"""
|
||||
return False
|
||||
|
||||
@property
|
||||
def supports_eplb(self) -> bool:
|
||||
return False
|
||||
@@ -162,20 +171,44 @@ class FusedMoEMethodBase(QuantizeMethodBase):
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: "RoutedExperts", # type: ignore[name-defined] # noqa: F821
|
||||
layer: "RoutedExperts",
|
||||
x: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: "SharedExperts | None",
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Apply the MoE operation using modular kernels.
|
||||
|
||||
Args:
|
||||
layer: RoutedExperts instance containing weight parameters
|
||||
x: Input tensor
|
||||
topk_weights: Expert weights from router
|
||||
topk_ids: Selected expert IDs from router
|
||||
shared_experts_input: Input for shared experts (if any)
|
||||
|
||||
Returns:
|
||||
Output tensor from routed experts
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def apply_monolithic(
|
||||
self,
|
||||
layer: "RoutedExperts", # type: ignore[name-defined] # noqa: F821
|
||||
layer: "RoutedExperts",
|
||||
x: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Apply the MoE operation using monolithic kernels.
|
||||
|
||||
Args:
|
||||
layer: RoutedExperts instance containing weight parameters
|
||||
x: Input tensor
|
||||
router_logits: Router logits (routing done internally)
|
||||
|
||||
Returns:
|
||||
Output tensor from routed experts
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
@@ -20,6 +21,11 @@ from vllm.model_executor.layers.fused_moe.runner.shared_experts import (
|
||||
SharedExperts,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts import (
|
||||
RoutedExperts,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -43,7 +49,7 @@ class FusedMoEModularMethod(FusedMoEMethodBase, CustomOp):
|
||||
|
||||
@staticmethod
|
||||
def make(
|
||||
moe_layer: torch.nn.Module,
|
||||
routed_experts: "RoutedExperts",
|
||||
old_quant_method: FusedMoEMethodBase,
|
||||
prepare_finalize: FusedMoEPrepareAndFinalizeModular,
|
||||
) -> "FusedMoEModularMethod":
|
||||
@@ -51,10 +57,18 @@ class FusedMoEModularMethod(FusedMoEMethodBase, CustomOp):
|
||||
old_quant_method,
|
||||
FusedMoEKernel(
|
||||
prepare_finalize,
|
||||
old_quant_method.select_gemm_impl(prepare_finalize, moe_layer),
|
||||
old_quant_method.select_gemm_impl(prepare_finalize, routed_experts),
|
||||
),
|
||||
)
|
||||
|
||||
@property
|
||||
def skip_forward_padding(self) -> bool:
|
||||
return self.old_quant_method.skip_forward_padding
|
||||
|
||||
@property
|
||||
def has_unpadded_output(self) -> bool:
|
||||
return self.old_quant_method.has_unpadded_output
|
||||
|
||||
@property
|
||||
def supports_eplb(self) -> bool:
|
||||
return self.old_quant_method.supports_eplb
|
||||
@@ -65,7 +79,7 @@ class FusedMoEModularMethod(FusedMoEMethodBase, CustomOp):
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
layer: "RoutedExperts",
|
||||
num_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
@@ -75,13 +89,13 @@ class FusedMoEModularMethod(FusedMoEMethodBase, CustomOp):
|
||||
raise NotImplementedError
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
self, layer: torch.nn.Module
|
||||
self, layer: "RoutedExperts"
|
||||
) -> FusedMoEQuantConfig | None:
|
||||
return self.moe_quant_config
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: "RoutedExperts", # type: ignore[name-defined] # noqa: F821
|
||||
layer: "RoutedExperts",
|
||||
x: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1097,8 +1097,9 @@ class FusedMoEKernelModularImpl:
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
):
|
||||
if shared_experts is not None:
|
||||
assert self.prepare_finalize.supports_async()
|
||||
assert shared_experts_input is not None
|
||||
shared_experts.apply(
|
||||
shared_experts(
|
||||
shared_experts_input,
|
||||
SharedExpertsOrder.MK_INTERNAL_OVERLAPPED,
|
||||
)
|
||||
@@ -1572,7 +1573,7 @@ class FusedMoEKernel:
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
router_logits: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
router_logits: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
|
||||
@@ -18,6 +18,7 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
fp8_w8a8_moe_quant_config,
|
||||
fp8_w8a16_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
FlashinferMoeBackend,
|
||||
get_flashinfer_moe_backend,
|
||||
@@ -423,7 +424,7 @@ def select_fp8_moe_backend(
|
||||
def convert_to_fp8_moe_kernel_format(
|
||||
fp8_backend: Fp8MoeBackend,
|
||||
# TODO(bnell): replace layer with weight_block_size
|
||||
layer: torch.nn.Module,
|
||||
layer: RoutedExperts,
|
||||
w13: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
|
||||
@@ -13,6 +13,7 @@ from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEConfig,
|
||||
RoutedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
maybe_make_prepare_finalize,
|
||||
@@ -1632,12 +1633,11 @@ def make_mxfp4_moe_quant_config(
|
||||
gemm1_clamp_limit=swiglu_limit,
|
||||
)
|
||||
elif mxfp4_backend == Mxfp4MoeBackend.HUMMING:
|
||||
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
get_humming_moe_quant_config,
|
||||
)
|
||||
|
||||
assert isinstance(layer, FusedMoE)
|
||||
assert isinstance(layer, RoutedExperts)
|
||||
return get_humming_moe_quant_config(
|
||||
layer,
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
@@ -1663,7 +1663,7 @@ def make_mxfp4_moe_kernel(
|
||||
experts_cls: type[mk.FusedMoEExperts],
|
||||
mxfp4_backend: Mxfp4MoeBackend,
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
layer: "RoutedExperts | None" = None,
|
||||
layer: RoutedExperts | None = None,
|
||||
) -> mk.FusedMoEKernel:
|
||||
"""Create a FusedMoEKernel for the given MXFP4 backend."""
|
||||
is_monolithic = issubclass(experts_cls, mk.FusedMoEExpertsMonolithic)
|
||||
|
||||
@@ -17,6 +17,7 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
nvfp4_moe_quant_config,
|
||||
nvfp4_w4a16_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
|
||||
prepare_nvfp4_moe_layer_for_fi_or_cutlass,
|
||||
prepare_nvfp4_moe_layer_for_flashinfer_cutedsl,
|
||||
@@ -335,7 +336,7 @@ def select_nvfp4_moe_backend(
|
||||
|
||||
def convert_to_nvfp4_moe_kernel_format(
|
||||
nvfp4_backend: NvFp4MoeBackend,
|
||||
layer: torch.nn.Module,
|
||||
layer: RoutedExperts,
|
||||
w13: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
w13_scale_2: torch.Tensor,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
|
||||
@@ -37,13 +36,11 @@ class AiterSharedRoutedFusedMoERouter(BaseRouter):
|
||||
eplb_state: EplbLayerState | None = None,
|
||||
scoring_func: str = "softmax",
|
||||
renormalize: bool = True,
|
||||
indices_type_getter: Callable[[], torch.dtype | None] | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
top_k=top_k,
|
||||
global_num_experts=global_num_experts,
|
||||
eplb_state=eplb_state,
|
||||
indices_type_getter=indices_type_getter,
|
||||
)
|
||||
self.renormalize = renormalize
|
||||
self.scoring_func = scoring_func
|
||||
|
||||
@@ -149,27 +149,16 @@ class BaseRouter(FusedMoERouter):
|
||||
top_k: int,
|
||||
global_num_experts: int,
|
||||
eplb_state: EplbLayerState | None = None,
|
||||
# TODO(bnell): Once the MK is constructed at layer init time, we
|
||||
# can make this a plain value instead of a callback.
|
||||
indices_type_getter: Callable[[], torch.dtype | None] | None = None,
|
||||
):
|
||||
"""
|
||||
Note: the indices dtype might not be available at router construction
|
||||
time, so we need to supply a callback to get it at runtime. This is
|
||||
because the indices type is supplied by modular kernels which are
|
||||
created after MoE layer/router construction.
|
||||
|
||||
Args:
|
||||
top_k: Number of experts to select per token
|
||||
global_num_experts: Total number of experts
|
||||
eplb_state: Optional EPLBLayerState for load balancing
|
||||
indices_type_getter: Optional callback to get indices dtype
|
||||
"""
|
||||
super().__init__()
|
||||
super().__init__(eplb_state=eplb_state)
|
||||
self.top_k = top_k
|
||||
self.global_num_experts = global_num_experts
|
||||
self.eplb_state = eplb_state
|
||||
self.indices_type_getter = indices_type_getter
|
||||
self.capture_fn: Callable[[torch.Tensor], None] | None = None
|
||||
|
||||
def set_capture_fn(self, capture_fn: Callable[[torch.Tensor], None] | None) -> None:
|
||||
@@ -189,12 +178,6 @@ class BaseRouter(FusedMoERouter):
|
||||
if eplb_state.should_record_tensor is None:
|
||||
raise ValueError("EPLB requires should_record_tensor != None")
|
||||
|
||||
def _get_indices_type(self) -> torch.dtype | None:
|
||||
"""Get the desired indices dtype from the getter function."""
|
||||
return (
|
||||
self.indices_type_getter() if self.indices_type_getter is not None else None
|
||||
)
|
||||
|
||||
def _apply_eplb_mapping(self, topk_ids: torch.Tensor) -> torch.Tensor:
|
||||
"""Apply EPLB mapping to convert logical expert IDs to physical expert IDs."""
|
||||
if self.eplb_state is not None:
|
||||
@@ -247,10 +230,11 @@ class BaseRouter(FusedMoERouter):
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def select_experts(
|
||||
def _select_experts(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
topk_indices_dtype: torch.dtype | None = None,
|
||||
*,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
@@ -260,10 +244,9 @@ class BaseRouter(FusedMoERouter):
|
||||
|
||||
This method implements the template method pattern:
|
||||
1. Validates EPLB state
|
||||
2. Gets indices type
|
||||
3. Calls _compute_routing() to get topk_weights and topk_ids
|
||||
4. Applies EPLB mapping if enabled
|
||||
5. Converts indices dtype if needed
|
||||
2. Calls _compute_routing() to get topk_weights and topk_ids
|
||||
3. Applies EPLB mapping if enabled
|
||||
4. Converts indices dtype if needed
|
||||
|
||||
Returns:
|
||||
(topk_weights, topk_ids)
|
||||
@@ -277,22 +260,19 @@ class BaseRouter(FusedMoERouter):
|
||||
# Step 1: Validate EPLB state
|
||||
self._validate_eplb_state()
|
||||
|
||||
# Step 2: Get indices type.
|
||||
indices_type = self._get_indices_type()
|
||||
|
||||
# Step 3: Compute routing (delegated to subclass)
|
||||
# Step 2: Compute routing (delegated to subclass)
|
||||
topk_weights, topk_ids = self._compute_routing(
|
||||
hidden_states, router_logits, indices_type, input_ids=input_ids
|
||||
hidden_states, router_logits, topk_indices_dtype, input_ids=input_ids
|
||||
)
|
||||
|
||||
# Capture logical ids before EPLB mapping.
|
||||
if self.capture_fn is not None:
|
||||
self.capture_fn(topk_ids)
|
||||
|
||||
# Step 4: Apply EPLB mapping
|
||||
# Step 3: Apply EPLB mapping
|
||||
topk_ids = self._apply_eplb_mapping(topk_ids)
|
||||
|
||||
# Step 5: Convert indices dtype
|
||||
topk_ids = self._convert_indices_dtype(topk_ids, indices_type)
|
||||
# Step 4: Convert indices dtype
|
||||
topk_ids = self._convert_indices_dtype(topk_ids, topk_indices_dtype)
|
||||
|
||||
return topk_weights, topk_ids
|
||||
|
||||
@@ -19,13 +19,11 @@ class CustomRoutingRouter(BaseRouter):
|
||||
custom_routing_function: Callable,
|
||||
eplb_state: EplbLayerState | None = None,
|
||||
renormalize: bool = True,
|
||||
indices_type_getter: Callable[[], torch.dtype | None] | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
top_k=top_k,
|
||||
global_num_experts=global_num_experts,
|
||||
eplb_state=eplb_state,
|
||||
indices_type_getter=indices_type_getter,
|
||||
)
|
||||
self.custom_routing_function = custom_routing_function
|
||||
self.renormalize = renormalize
|
||||
|
||||
@@ -5,6 +5,7 @@ from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.eplb.eplb_state import EplbLayerState
|
||||
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
|
||||
|
||||
|
||||
@@ -14,6 +15,10 @@ class FusedMoERouter(ABC):
|
||||
method that is used for routing hidden states based on router logits.
|
||||
"""
|
||||
|
||||
def __init__(self, eplb_state: EplbLayerState | None = None):
|
||||
self._routing_replay_out: torch.Tensor | None = None
|
||||
self.eplb_state = eplb_state
|
||||
|
||||
@abstractmethod
|
||||
def set_capture_fn(
|
||||
self,
|
||||
@@ -27,10 +32,21 @@ class FusedMoERouter(ABC):
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def _select_experts(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
topk_indices_dtype: torch.dtype | None = None,
|
||||
*,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
raise NotImplementedError
|
||||
|
||||
def select_experts(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
topk_indices_dtype: torch.dtype | None = None,
|
||||
*,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
@@ -47,4 +63,19 @@ class FusedMoERouter(ABC):
|
||||
equivalent to global logical ids, so should be compatible with
|
||||
plain MoE implementations without redundant experts.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
topk_weights, topk_ids = self._select_experts(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
topk_indices_dtype=topk_indices_dtype,
|
||||
input_ids=input_ids,
|
||||
)
|
||||
|
||||
# Write routing data for non-monolithic path (Triton, etc.)
|
||||
# (set by bind_routing_capture_to_model during capturer init)
|
||||
if self._routing_replay_out is not None:
|
||||
self._routing_replay_out[: topk_ids.shape[0]].copy_(
|
||||
topk_ids.to(torch.int16)
|
||||
)
|
||||
|
||||
return topk_weights, topk_ids
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import functools
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
@@ -300,7 +299,6 @@ class FusedTopKBiasRouter(BaseRouter):
|
||||
renormalize: bool = True,
|
||||
routed_scaling_factor: float = 1.0,
|
||||
eplb_state: EplbLayerState | None = None,
|
||||
indices_type_getter: Callable[[], torch.dtype | None] | None = None,
|
||||
*,
|
||||
scoring_func: str = "sigmoid",
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
@@ -309,7 +307,6 @@ class FusedTopKBiasRouter(BaseRouter):
|
||||
top_k=top_k,
|
||||
global_num_experts=global_num_experts,
|
||||
eplb_state=eplb_state,
|
||||
indices_type_getter=indices_type_getter,
|
||||
)
|
||||
self.e_score_correction_bias = e_score_correction_bias
|
||||
self.renormalize = renormalize
|
||||
|
||||
@@ -123,13 +123,11 @@ class FusedTopKRouter(BaseRouter):
|
||||
scoring_func: str = "softmax",
|
||||
renormalize: bool = True,
|
||||
eplb_state: EplbLayerState | None = None,
|
||||
indices_type_getter: Callable[[], torch.dtype | None] | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
top_k=top_k,
|
||||
global_num_experts=global_num_experts,
|
||||
eplb_state=eplb_state,
|
||||
indices_type_getter=indices_type_getter,
|
||||
)
|
||||
self.renormalize = renormalize
|
||||
self.scoring_func = scoring_func
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable
|
||||
from functools import partial
|
||||
|
||||
import torch
|
||||
@@ -259,13 +258,11 @@ class GroupedTopKRouter(BaseRouter):
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
num_fused_shared_experts: int = 0,
|
||||
eplb_state: EplbLayerState | None = None,
|
||||
indices_type_getter: Callable[[], torch.dtype | None] | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
top_k=top_k,
|
||||
global_num_experts=global_num_experts,
|
||||
eplb_state=eplb_state,
|
||||
indices_type_getter=indices_type_getter,
|
||||
)
|
||||
self.num_expert_group = num_expert_group
|
||||
self.topk_group = topk_group
|
||||
|
||||
@@ -41,7 +41,6 @@ def create_fused_moe_router(
|
||||
top_k: int,
|
||||
global_num_experts: int,
|
||||
renormalize: bool = True,
|
||||
indices_type_getter: Callable[[], torch.dtype | None] | None = None,
|
||||
# grouped topk parameters
|
||||
use_grouped_topk: bool = False,
|
||||
num_expert_group: int | None = None,
|
||||
@@ -77,7 +76,6 @@ def create_fused_moe_router(
|
||||
top_k: Number of experts to select per token
|
||||
global_num_experts: Total number of experts in the model
|
||||
renormalize: Whether to renormalize the routing weights
|
||||
indices_type_getter: Function to get the desired indices dtype
|
||||
routing_method_type: Optional explicit routing method type
|
||||
|
||||
Grouped topk arguments:
|
||||
@@ -116,7 +114,6 @@ def create_fused_moe_router(
|
||||
top_k=top_k,
|
||||
global_num_experts=global_num_experts,
|
||||
eplb_state=eplb_state,
|
||||
indices_type_getter=indices_type_getter,
|
||||
)
|
||||
|
||||
if zero_expert_type is not None:
|
||||
@@ -136,7 +133,6 @@ def create_fused_moe_router(
|
||||
scoring_func=scoring_func,
|
||||
renormalize=renormalize,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
indices_type_getter=indices_type_getter,
|
||||
)
|
||||
|
||||
if use_grouped_topk:
|
||||
@@ -157,7 +153,6 @@ def create_fused_moe_router(
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
num_fused_shared_experts=num_fused_shared_experts,
|
||||
indices_type_getter=indices_type_getter,
|
||||
)
|
||||
if (
|
||||
grouped_topk_router.routing_method_type != RoutingMethodType.Unspecified
|
||||
@@ -179,7 +174,6 @@ def create_fused_moe_router(
|
||||
eplb_state=eplb_state,
|
||||
custom_routing_function=custom_routing_function,
|
||||
renormalize=renormalize,
|
||||
indices_type_getter=indices_type_getter,
|
||||
)
|
||||
|
||||
assert scoring_func in ["sigmoid", "softmax", "sqrtsoftplus"]
|
||||
@@ -192,7 +186,6 @@ def create_fused_moe_router(
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
renormalize=renormalize,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
indices_type_getter=indices_type_getter,
|
||||
scoring_func=scoring_func,
|
||||
hash_indices_table=hash_indices_table,
|
||||
)
|
||||
@@ -209,7 +202,6 @@ def create_fused_moe_router(
|
||||
num_fused_shared_experts=num_fused_shared_experts,
|
||||
renormalize=renormalize,
|
||||
scoring_func=scoring_func,
|
||||
indices_type_getter=indices_type_getter,
|
||||
)
|
||||
|
||||
return FusedTopKRouter(
|
||||
@@ -218,5 +210,4 @@ def create_fused_moe_router(
|
||||
eplb_state=eplb_state,
|
||||
renormalize=renormalize,
|
||||
scoring_func=scoring_func,
|
||||
indices_type_getter=indices_type_getter,
|
||||
)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
@@ -314,13 +313,11 @@ class RoutingSimulatorRouter(BaseRouter):
|
||||
top_k: int,
|
||||
global_num_experts: int,
|
||||
eplb_state: EplbLayerState | None = None,
|
||||
indices_type_getter: Callable[[], torch.dtype | None] | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
top_k=top_k,
|
||||
global_num_experts=global_num_experts,
|
||||
eplb_state=eplb_state,
|
||||
indices_type_getter=indices_type_getter,
|
||||
)
|
||||
|
||||
@property
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.eplb.eplb_state import EplbLayerState
|
||||
@@ -39,13 +37,11 @@ class ZeroExpertRouter(BaseRouter):
|
||||
renormalize: bool = False,
|
||||
routed_scaling_factor: float = 1.0,
|
||||
eplb_state: EplbLayerState | None = None,
|
||||
indices_type_getter: Callable[[], torch.dtype | None] | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
top_k=top_k,
|
||||
global_num_experts=global_num_experts,
|
||||
eplb_state=eplb_state,
|
||||
indices_type_getter=indices_type_getter,
|
||||
)
|
||||
self.e_score_correction_bias = e_score_correction_bias
|
||||
self.num_logical_experts = num_logical_experts
|
||||
|
||||
@@ -1,28 +1,39 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Iterable
|
||||
from contextlib import nullcontext
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.config.parallel import ExpertPlacementStrategy
|
||||
from vllm.distributed import (
|
||||
get_ep_group,
|
||||
get_pcp_group,
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from vllm.distributed.eplb.eplb_state import EplbLayerState
|
||||
from vllm.forward_context import (
|
||||
ForwardContext,
|
||||
get_forward_context,
|
||||
is_forward_context_available,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
|
||||
FusedMoEMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe_modular_method import (
|
||||
FusedMoEModularMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts import (
|
||||
RoutedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.fused_moe_router import (
|
||||
FusedMoERouter,
|
||||
)
|
||||
@@ -43,8 +54,23 @@ from vllm.utils.torch_utils import (
|
||||
direct_register_custom_op,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
def get_layer_from_name(layer_name: str) -> torch.nn.Module:
|
||||
|
||||
def register_layer_for_moe_forward_op(
|
||||
vllm_config: VllmConfig,
|
||||
layer: "MoERunner",
|
||||
):
|
||||
# For smuggling this layer into the fused moe custom op
|
||||
prefix = layer.layer_name
|
||||
compilation_config = vllm_config.compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError("Duplicate layer name: {}".format(prefix))
|
||||
compilation_config.static_forward_context[prefix] = layer
|
||||
compilation_config.static_all_moe_layers.append(prefix)
|
||||
|
||||
|
||||
def get_layer_from_name(layer_name: str) -> MoERunnerInterface:
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
if not _USE_LAYERNAME and layer_name == "from_forward_context":
|
||||
all_moe_layers = forward_context.all_moe_layers
|
||||
@@ -58,7 +84,9 @@ def get_layer_from_name(layer_name: str) -> torch.nn.Module:
|
||||
)
|
||||
layer_name = all_moe_layers[moe_layer_index]
|
||||
forward_context.moe_layer_index += 1
|
||||
return forward_context.no_compile_layers[layer_name]
|
||||
layer = forward_context.no_compile_layers[layer_name]
|
||||
assert isinstance(layer, MoERunnerInterface)
|
||||
return layer
|
||||
|
||||
|
||||
# On torch >= 2.11, layer_name is a hoisted LayerName opaque object;
|
||||
@@ -96,8 +124,7 @@ def _moe_forward(
|
||||
hidden_dim_unpadded: int,
|
||||
) -> torch.Tensor:
|
||||
layer = get_layer_from_name(_resolve_layer_name(layer_name))
|
||||
return layer.runner._forward_impl(
|
||||
layer,
|
||||
return layer._forward_impl(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
shared_experts_input,
|
||||
@@ -131,8 +158,7 @@ def _moe_forward_shared(
|
||||
hidden_dim_unpadded: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
layer = get_layer_from_name(_resolve_layer_name(layer_name))
|
||||
return layer.runner._forward_impl(
|
||||
layer,
|
||||
return layer._forward_impl(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
shared_experts_input,
|
||||
@@ -220,12 +246,12 @@ class MoERunner(MoERunnerInterface):
|
||||
layer_name: str,
|
||||
moe_config: FusedMoEConfig,
|
||||
router: FusedMoERouter,
|
||||
routed_input_transform: torch.nn.Module | None,
|
||||
gate: torch.nn.Module | None,
|
||||
shared_experts: torch.nn.Module | None,
|
||||
quant_method: FusedMoEMethodBase,
|
||||
enable_dbo: bool,
|
||||
routed_experts: RoutedExperts,
|
||||
enable_dbo: bool = False,
|
||||
gate: torch.nn.Module | None = None,
|
||||
shared_experts: torch.nn.Module | None = None,
|
||||
shared_expert_gate: torch.nn.Module | None = None,
|
||||
routed_input_transform: torch.nn.Module | None = None,
|
||||
routed_output_transform: torch.nn.Module | None = None,
|
||||
routed_scaling_factor: float = 1.0,
|
||||
):
|
||||
@@ -237,7 +263,7 @@ class MoERunner(MoERunnerInterface):
|
||||
self.routed_scaling_factor = routed_scaling_factor
|
||||
self.gate = gate
|
||||
self.shared_expert_gate = shared_expert_gate
|
||||
self._quant_method = quant_method
|
||||
self.routed_experts = routed_experts
|
||||
self.enable_dbo = enable_dbo
|
||||
|
||||
# When both gates are present and FSE is enabled, fuse their
|
||||
@@ -250,23 +276,22 @@ class MoERunner(MoERunnerInterface):
|
||||
|
||||
self._shared_experts: SharedExperts | None = None
|
||||
if shared_experts is not None:
|
||||
can_overlap = lambda: self._quant_method.mk_can_overlap_shared_experts
|
||||
self._shared_experts = SharedExperts(
|
||||
shared_experts,
|
||||
moe_config=moe_config,
|
||||
# Note: For now we must pass quant_method along to SharedExperts so it
|
||||
# can property determine where the shared experts are supposed to be
|
||||
# called, i.e. by a MK or by the MoERunner.
|
||||
# Once the MK can be created upfront, we can just pass in the proper
|
||||
# flags derived from the quant_method's MK.
|
||||
quant_method=quant_method,
|
||||
enable_dbo=enable_dbo,
|
||||
mk_can_overlap_shared_experts=can_overlap,
|
||||
)
|
||||
|
||||
# Needed for string -> FusedMoE layer lookup in custom ops.
|
||||
# Needed for string -> MoERunner layer lookup in custom ops.
|
||||
self.layer_name = layer_name
|
||||
|
||||
self._forward_entry = self._select_forward()
|
||||
|
||||
# For smuggling this layer into the fused moe custom op
|
||||
register_layer_for_moe_forward_op(get_current_vllm_config(), self)
|
||||
|
||||
def _select_forward(self) -> Callable:
|
||||
if current_platform.is_tpu() or current_platform.is_cpu():
|
||||
# TODO: Once the OOM issue for the TPU backend is resolved, we
|
||||
@@ -284,11 +309,20 @@ class MoERunner(MoERunnerInterface):
|
||||
def shared_experts(self) -> SharedExperts | None:
|
||||
return self._shared_experts
|
||||
|
||||
# TODO(bnell): temporary hack, do not call this method.
|
||||
@property
|
||||
def is_internal_router(self) -> bool:
|
||||
return self.gate is not None
|
||||
|
||||
# TODO(bnell): Temporary hack. Get rid of this.
|
||||
def _replace_quant_method(self, quant_method: FusedMoEMethodBase):
|
||||
self.routed_experts._replace_quant_method(quant_method)
|
||||
|
||||
# TODO(bnell): Hack for elastic_ep. Get rid of this
|
||||
def _set_moe_config(self, new_moe_config: FusedMoEConfig):
|
||||
self.moe_config = new_moe_config
|
||||
self.routed_experts._set_moe_config(new_moe_config)
|
||||
if self._shared_experts is not None:
|
||||
self._shared_experts._quant_method = quant_method
|
||||
self._quant_method = quant_method
|
||||
self._shared_experts._set_moe_config(new_moe_config)
|
||||
|
||||
def _maybe_fuse_gate_weights(self):
|
||||
"""Fuse router and shared expert gate weights on first call.
|
||||
@@ -304,8 +338,9 @@ class MoERunner(MoERunnerInterface):
|
||||
dim=0,
|
||||
)
|
||||
|
||||
def is_internal_router(self) -> bool:
|
||||
return self.gate is not None
|
||||
@property
|
||||
def _quant_method(self) -> FusedMoEMethodBase:
|
||||
return self.routed_experts.quant_method
|
||||
|
||||
def apply_routed_input_transform(
|
||||
self, hidden_states: torch.Tensor
|
||||
@@ -396,9 +431,9 @@ class MoERunner(MoERunnerInterface):
|
||||
def _maybe_reduce_final_output(
|
||||
self,
|
||||
states: torch.Tensor,
|
||||
trunc_size: int,
|
||||
trunc_size: int | None,
|
||||
) -> torch.Tensor:
|
||||
"""Truncate padded dimensions and all-reduce the combined output.
|
||||
"""All-reduce the combined output if needed.
|
||||
|
||||
This is the "late" all-reduce path. When neither fused nor shared
|
||||
output was individually reduced, the combined sum is all-reduced
|
||||
@@ -415,7 +450,7 @@ class MoERunner(MoERunnerInterface):
|
||||
):
|
||||
states = tensor_model_parallel_all_reduce(states)
|
||||
|
||||
return states[..., :trunc_size]
|
||||
return states[..., :trunc_size] if trunc_size is not None else states
|
||||
|
||||
def _encode_layer_name(self) -> str | LayerName:
|
||||
if _USE_LAYERNAME:
|
||||
@@ -428,34 +463,11 @@ class MoERunner(MoERunnerInterface):
|
||||
return "from_forward_context"
|
||||
return self.layer_name
|
||||
|
||||
def _trtllm_mxfp4_unpadded_dim(self) -> int:
|
||||
"""Return ``hidden_dim_unpadded`` when the active backend is TRT-LLM
|
||||
MXFP4 (whose kernel writes narrower than the padded
|
||||
``hidden_states.shape[-1]``), else 0. Other MXFP4 backends (notably
|
||||
Cutlass MXFP4 MXFP8) write the full padded width, so
|
||||
``moe_config.hidden_dim_unpadded`` alone is insufficient: it encodes
|
||||
the model's logical hidden, not whether the kernel narrows. Computed
|
||||
caller-side and passed as an op arg; doing the isinstance check
|
||||
inside the fake would specialize per ``layer_name`` and break
|
||||
subgraph dedup for identical-architecture models (e.g. Phi-MoE).
|
||||
"""
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe import (
|
||||
TrtLlmMxfp4ExpertsBase,
|
||||
)
|
||||
|
||||
moe_kernel = getattr(self._quant_method, "moe_kernel", None)
|
||||
fused_experts = getattr(
|
||||
getattr(moe_kernel, "impl", None), "fused_experts", None
|
||||
)
|
||||
if isinstance(fused_experts, TrtLlmMxfp4ExpertsBase):
|
||||
return self.moe_config.hidden_dim_unpadded or self.moe_config.hidden_dim
|
||||
return 0
|
||||
|
||||
def _maybe_pad_hidden_states(
|
||||
self,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, int]:
|
||||
) -> tuple[torch.Tensor, int | None, int | None]:
|
||||
"""Pad hidden_states to moe_config.hidden_dim and compute the
|
||||
original dimension for later truncation.
|
||||
|
||||
@@ -467,11 +479,12 @@ class MoERunner(MoERunnerInterface):
|
||||
shared_experts_hidden_dim = (
|
||||
shared_experts_input.shape[-1] if shared_experts_input is not None else 0
|
||||
)
|
||||
transformed_hidden_dim = hidden_states.shape[-1]
|
||||
transformed_hidden_dim: int | None = hidden_states.shape[-1]
|
||||
if (
|
||||
not self._quant_method.skip_forward_padding
|
||||
and self.moe_config.hidden_dim != transformed_hidden_dim
|
||||
):
|
||||
assert transformed_hidden_dim is not None
|
||||
hidden_states = F.pad(
|
||||
hidden_states,
|
||||
(0, self.moe_config.hidden_dim - transformed_hidden_dim),
|
||||
@@ -479,12 +492,33 @@ class MoERunner(MoERunnerInterface):
|
||||
value=0.0,
|
||||
)
|
||||
|
||||
if self.routed_output_transform is not None and shared_experts_hidden_dim > 0:
|
||||
orig_hidden_dims = shared_experts_hidden_dim
|
||||
else:
|
||||
orig_hidden_dims = transformed_hidden_dim
|
||||
# Truncation sizes for stripping kernel padding from the output.
|
||||
# None means no truncation needed (no padding was applied).
|
||||
#
|
||||
# Two truncation points exist in forward():
|
||||
# pre_xform: applied to fused_output BEFORE routed_output_transform
|
||||
# post_xform: applied to the final result AFTER all-reduce
|
||||
#
|
||||
# Latent MoE with shared experts (NemotronH):
|
||||
# - pre_xform strips padding from the latent dim so
|
||||
# routed_output_transform receives the correct input size
|
||||
# - post_xform truncates to shared_experts_hidden_dim (full hidden)
|
||||
# after shared + routed outputs are combined and all-reduced
|
||||
#
|
||||
# Standard MoE / MoE without transforms (GPT-OSS, Mixtral):
|
||||
# - pre_xform is None (no early truncation)
|
||||
# - post_xform strips padding after all-reduce (or None if unpadded)
|
||||
if transformed_hidden_dim == hidden_states.shape[-1]:
|
||||
transformed_hidden_dim = None
|
||||
|
||||
return hidden_states, orig_hidden_dims
|
||||
if self.routed_output_transform is not None and shared_experts_hidden_dim > 0:
|
||||
pre_xform_trunc_size = transformed_hidden_dim
|
||||
post_xform_trunc_size = shared_experts_hidden_dim
|
||||
else:
|
||||
pre_xform_trunc_size = None
|
||||
post_xform_trunc_size = transformed_hidden_dim
|
||||
|
||||
return hidden_states, pre_xform_trunc_size, post_xform_trunc_size
|
||||
|
||||
def _maybe_apply_shared_experts(
|
||||
self,
|
||||
@@ -493,11 +527,10 @@ class MoERunner(MoERunnerInterface):
|
||||
):
|
||||
if self._shared_experts is not None:
|
||||
assert shared_experts_input is not None
|
||||
self._shared_experts.apply(shared_experts_input, order)
|
||||
self._shared_experts(shared_experts_input, order)
|
||||
|
||||
def _apply_quant_method(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
@@ -513,24 +546,23 @@ class MoERunner(MoERunnerInterface):
|
||||
shared_experts_input, SharedExpertsOrder.NO_OVERLAP
|
||||
)
|
||||
|
||||
if self._quant_method.is_monolithic:
|
||||
fused_out = self._quant_method.apply_monolithic(
|
||||
layer=layer,
|
||||
if self.routed_experts.quant_method.is_monolithic:
|
||||
# Monolithic kernels: pass router_logits to routed_experts
|
||||
fused_out = self.routed_experts.forward_monolithic(
|
||||
x=hidden_states,
|
||||
router_logits=router_logits,
|
||||
input_ids=input_ids,
|
||||
)
|
||||
else:
|
||||
# Modular kernels: select experts first, then call routed_experts
|
||||
topk_weights, topk_ids = self.router.select_experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
topk_indices_dtype=self._quant_method.topk_indices_dtype,
|
||||
input_ids=input_ids,
|
||||
)
|
||||
|
||||
# Passing shared_experts_input in case SharedExpertsOrder is
|
||||
# MK_INTERNAL_OVERLAPPED.
|
||||
fused_out = self._quant_method.apply(
|
||||
layer=layer,
|
||||
fused_out = self.routed_experts.forward_modular(
|
||||
x=hidden_states,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
@@ -627,12 +659,13 @@ class MoERunner(MoERunnerInterface):
|
||||
# `moe_config.hidden_dim`, e.g. after `align_trtllm_fp4_moe_hidden_dim_for_fi`
|
||||
# so routed output can be trimmed before
|
||||
# shared+routed add / latent up proj if needed.
|
||||
routed_hidden_dim = hidden_states.shape[-1]
|
||||
hidden_states, og_hidden_dim = self._maybe_pad_hidden_states(
|
||||
shared_experts_input,
|
||||
hidden_states,
|
||||
|
||||
hidden_states, og_hidden_dim_pre_xform, og_hidden_dim_post_xform = (
|
||||
self._maybe_pad_hidden_states(
|
||||
shared_experts_input,
|
||||
hidden_states,
|
||||
)
|
||||
)
|
||||
hidden_dim_was_padded = hidden_states.shape[-1] > routed_hidden_dim
|
||||
|
||||
result = self._forward_entry(
|
||||
hidden_states,
|
||||
@@ -640,7 +673,9 @@ class MoERunner(MoERunnerInterface):
|
||||
shared_experts_input,
|
||||
input_ids,
|
||||
self._encode_layer_name(),
|
||||
self._trtllm_mxfp4_unpadded_dim(),
|
||||
self.moe_config.hidden_dim_unpadded
|
||||
if self._quant_method.has_unpadded_output
|
||||
else 0,
|
||||
)
|
||||
|
||||
#
|
||||
@@ -654,10 +689,9 @@ class MoERunner(MoERunnerInterface):
|
||||
|
||||
# Extract outputs from result
|
||||
shared_output, fused_output = _unpack(result)
|
||||
if (
|
||||
shared_output is not None or self.routed_output_transform is not None
|
||||
) and hidden_dim_was_padded:
|
||||
fused_output = fused_output[..., :routed_hidden_dim]
|
||||
|
||||
if og_hidden_dim_pre_xform is not None:
|
||||
fused_output = fused_output[..., :og_hidden_dim_pre_xform]
|
||||
|
||||
# If combine kernel already reduced fused, reduce shared to match.
|
||||
# See note above re: the two all-reduce points.
|
||||
@@ -675,7 +709,7 @@ class MoERunner(MoERunnerInterface):
|
||||
else:
|
||||
result = fused_output
|
||||
|
||||
result = self._maybe_reduce_final_output(result, og_hidden_dim)
|
||||
result = self._maybe_reduce_final_output(result, og_hidden_dim_post_xform)
|
||||
|
||||
return self._maybe_add_zero_expert_output(result)
|
||||
|
||||
@@ -687,7 +721,6 @@ class MoERunner(MoERunnerInterface):
|
||||
|
||||
def _maybe_dispatch(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
@@ -723,7 +756,7 @@ class MoERunner(MoERunnerInterface):
|
||||
self,
|
||||
shared_output: torch.Tensor | None,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor | None]:
|
||||
) -> torch.Tensor | tuple[torch.Tensor | None, torch.Tensor]:
|
||||
if self.do_naive_dispatch_combine:
|
||||
hidden_states = get_ep_group().combine(
|
||||
hidden_states, self.moe_config.is_sequence_parallel
|
||||
@@ -743,7 +776,6 @@ class MoERunner(MoERunnerInterface):
|
||||
|
||||
def _forward_impl(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
@@ -762,7 +794,7 @@ class MoERunner(MoERunnerInterface):
|
||||
Returns a single tensor of combined fused and shared output (if present).
|
||||
"""
|
||||
# TODO(bnell): this can be removed after MK migration is complete.
|
||||
layer.ensure_moe_quant_config_init()
|
||||
self.routed_experts._ensure_moe_quant_config_init()
|
||||
|
||||
# Sync aux and main stream for shared expert multi-stream overlap.
|
||||
self._maybe_sync_shared_experts_stream(shared_experts_input)
|
||||
@@ -782,13 +814,11 @@ class MoERunner(MoERunnerInterface):
|
||||
# #32567 lands and the remaining kernels are made MKs. The PCP
|
||||
# code will probably remain
|
||||
hidden_states, router_logits = self._maybe_dispatch(
|
||||
layer,
|
||||
hidden_states,
|
||||
router_logits,
|
||||
)
|
||||
|
||||
shared_output, hidden_states = self._apply_quant_method(
|
||||
layer=layer,
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
shared_experts_input=shared_experts_input,
|
||||
@@ -799,3 +829,148 @@ class MoERunner(MoERunnerInterface):
|
||||
shared_output,
|
||||
hidden_states,
|
||||
)
|
||||
|
||||
#########################################################
|
||||
#
|
||||
# Old methods from FusedMoE layer. Remove when possible.
|
||||
#
|
||||
#########################################################
|
||||
|
||||
# Note: maybe_init_modular_kernel should only be called by
|
||||
# prepare_communication_buffer_for_model.
|
||||
# This is called after all weight loading and post-processing, so it
|
||||
# should be safe to swap out the quant_method.
|
||||
def maybe_init_modular_kernel(self) -> None:
|
||||
# NOTE(rob): WIP refactor. For quant methods that own the MK
|
||||
# we create the MK during process_weights_after_loading.
|
||||
if (
|
||||
self.routed_experts.quant_method.supports_internal_mk
|
||||
or self.routed_experts.quant_method.is_monolithic
|
||||
):
|
||||
return None
|
||||
|
||||
self.routed_experts._ensure_moe_quant_config_init()
|
||||
# routing_tables only needed for round-robin expert placement with
|
||||
# DeepEP all2all backend.
|
||||
routing_tables = self._expert_routing_tables()
|
||||
|
||||
if isinstance(self.routed_experts.quant_method, FusedMoEModularMethod):
|
||||
base_quant_method = self.routed_experts.quant_method.old_quant_method
|
||||
else:
|
||||
base_quant_method = self.routed_experts.quant_method
|
||||
|
||||
prepare_finalize = base_quant_method.maybe_make_prepare_finalize(
|
||||
routing_tables=routing_tables
|
||||
)
|
||||
if prepare_finalize is not None:
|
||||
logger.debug(
|
||||
"%s for %s(%s)", prepare_finalize.__class__.__name__, self, id(self)
|
||||
)
|
||||
self._replace_quant_method(
|
||||
FusedMoEModularMethod.make(
|
||||
self.routed_experts,
|
||||
base_quant_method,
|
||||
prepare_finalize,
|
||||
)
|
||||
)
|
||||
|
||||
#
|
||||
# Properties
|
||||
#
|
||||
|
||||
@property
|
||||
def layer_id(self):
|
||||
# Delayed import to avoid circular dependency
|
||||
from vllm.model_executor.models.utils import extract_layer_index
|
||||
|
||||
return extract_layer_index(self.layer_name)
|
||||
|
||||
#
|
||||
# Attributes still needed by models
|
||||
#
|
||||
|
||||
@property
|
||||
def is_monolithic(self) -> bool:
|
||||
return self.routed_experts.quant_method.is_monolithic
|
||||
|
||||
@property
|
||||
def activation(self) -> MoEActivation:
|
||||
return self.routed_experts.activation
|
||||
|
||||
#
|
||||
# Expert maps
|
||||
#
|
||||
|
||||
@property
|
||||
def expert_map_manager(self):
|
||||
"""Forward to routed_experts.expert_map_manager for backward compatibility."""
|
||||
return self.routed_experts.expert_map_manager
|
||||
|
||||
@property
|
||||
def expert_placement_strategy(self) -> ExpertPlacementStrategy:
|
||||
return self.expert_map_manager.placement_strategy
|
||||
|
||||
@property
|
||||
def expert_global_to_physical(self) -> torch.Tensor | None:
|
||||
tables = self.expert_map_manager.routing_tables
|
||||
return tables[0] if tables else None
|
||||
|
||||
@property
|
||||
def expert_physical_to_global(self) -> torch.Tensor | None:
|
||||
"""Routing table: physical expert ID to global expert ID."""
|
||||
tables = self.expert_map_manager.routing_tables
|
||||
return tables[1] if tables else None
|
||||
|
||||
@property
|
||||
def expert_local_to_global(self) -> torch.Tensor | None:
|
||||
"""Routing table: local expert ID to global expert ID."""
|
||||
tables = self.expert_map_manager.routing_tables
|
||||
return tables[2] if tables else None
|
||||
|
||||
@property
|
||||
def expert_map(self) -> torch.Tensor | None:
|
||||
return self.routed_experts.expert_map
|
||||
|
||||
def _expert_routing_tables(
|
||||
self,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None:
|
||||
return self.routed_experts._expert_routing_tables()
|
||||
|
||||
def update_expert_map(self):
|
||||
self.routed_experts.update_expert_map()
|
||||
|
||||
def _map_global_expert_id_to_local_expert_id(self, expert_id: int) -> int:
|
||||
"""Map global expert ID to local expert ID."""
|
||||
return self.routed_experts._map_global_expert_id_to_local_expert_id(expert_id)
|
||||
|
||||
def get_expert_weights(self) -> Iterable[torch.Tensor]:
|
||||
return self.routed_experts.get_expert_weights()
|
||||
|
||||
#
|
||||
# EPLB
|
||||
#
|
||||
|
||||
@property
|
||||
def eplb_state(self) -> EplbLayerState | None:
|
||||
return self.router.eplb_state
|
||||
|
||||
def set_eplb_state(
|
||||
self,
|
||||
moe_layer_idx: int,
|
||||
expert_load_view: torch.Tensor,
|
||||
logical_to_physical_map: torch.Tensor,
|
||||
logical_replica_count: torch.Tensor,
|
||||
) -> None:
|
||||
"""
|
||||
Register the EPLB state in this layer.
|
||||
|
||||
This is used later in forward pass, where we get the expert mapping
|
||||
and record the load metrics in `expert_load_view`.
|
||||
"""
|
||||
if self.router.eplb_state is not None:
|
||||
self.router.eplb_state.set_layer_state(
|
||||
moe_layer_idx,
|
||||
expert_load_view,
|
||||
logical_to_physical_map,
|
||||
logical_replica_count,
|
||||
)
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config.parallel import ExpertPlacementStrategy
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
|
||||
FusedMoEMethodBase,
|
||||
)
|
||||
@@ -22,6 +25,11 @@ class MoERunnerInterface(PluggableLayer, ABC):
|
||||
expert routing, and managing tensor parallel operations.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
# HACK
|
||||
self._already_called_process_weights_after_loading = True
|
||||
|
||||
@abstractmethod
|
||||
def forward(
|
||||
self,
|
||||
@@ -31,16 +39,108 @@ class MoERunnerInterface(PluggableLayer, ABC):
|
||||
) -> torch.Tensor:
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def shared_experts(self) -> SharedExperts | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def is_internal_router(self) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def shared_experts(self) -> SharedExperts | None:
|
||||
def _quant_method(self) -> FusedMoEMethodBase:
|
||||
raise NotImplementedError
|
||||
|
||||
# TODO(bnell): temporary hack, do not call this method.
|
||||
# Temporary hack
|
||||
@abstractmethod
|
||||
def _replace_quant_method(self, quant_method: FusedMoEMethodBase):
|
||||
raise NotImplementedError
|
||||
|
||||
########################################################################
|
||||
#
|
||||
# FusedMoE layer methods
|
||||
#
|
||||
########################################################################
|
||||
|
||||
@abstractmethod
|
||||
def maybe_init_modular_kernel(self) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def layer_id(self):
|
||||
raise NotImplementedError
|
||||
|
||||
#
|
||||
# Attributes still needed by models
|
||||
#
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def is_monolithic(self) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def activation(self) -> MoEActivation:
|
||||
raise NotImplementedError
|
||||
|
||||
#
|
||||
# Expert maps
|
||||
#
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def expert_placement_strategy(self) -> ExpertPlacementStrategy:
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def expert_global_to_physical(self) -> torch.Tensor | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def expert_physical_to_global(self) -> torch.Tensor | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def expert_local_to_global(self) -> torch.Tensor | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def expert_map(self) -> torch.Tensor | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def _expert_routing_tables(
|
||||
self,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def update_expert_map(self):
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def _map_global_expert_id_to_local_expert_id(self, expert_id: int) -> int:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def get_expert_weights(self) -> Iterable[torch.Tensor]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def set_eplb_state(
|
||||
self,
|
||||
moe_layer_idx: int,
|
||||
expert_load_view: torch.Tensor,
|
||||
logical_to_physical_map: torch.Tensor,
|
||||
logical_replica_count: torch.Tensor,
|
||||
) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable
|
||||
from enum import IntEnum
|
||||
|
||||
import torch
|
||||
@@ -9,9 +10,6 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import (
|
||||
aux_stream,
|
||||
@@ -38,21 +36,15 @@ class SharedExpertsOrder(IntEnum):
|
||||
MULTI_STREAM_OVERLAPPED = (3,)
|
||||
|
||||
|
||||
class SharedExperts:
|
||||
class SharedExperts(torch.nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_method: QuantizeMethodBase,
|
||||
enable_dbo: bool,
|
||||
mk_can_overlap_shared_experts: Callable[[], bool],
|
||||
):
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
|
||||
FusedMoEMethodBase,
|
||||
)
|
||||
|
||||
# quant_method must be a FusedMoEMethodBase but we can't use the type
|
||||
# due to circular imports.
|
||||
assert isinstance(quant_method, FusedMoEMethodBase)
|
||||
super().__init__()
|
||||
|
||||
# The SharedExperts need to handle DBO since they can be called from
|
||||
# an MK's finalize method. We keep a list of outputs indexed by current
|
||||
@@ -62,7 +54,8 @@ class SharedExperts:
|
||||
self._output: list[torch.Tensor | None] = [None, None]
|
||||
self._layer = layer
|
||||
self._moe_config = moe_config
|
||||
self._quant_method = quant_method
|
||||
|
||||
self._mk_can_overlap_shared_experts = mk_can_overlap_shared_experts
|
||||
|
||||
# Allow disabling of the separate shared experts stream for
|
||||
# debug purposes.
|
||||
@@ -78,6 +71,10 @@ class SharedExperts:
|
||||
if self._stream is not None:
|
||||
logger.debug_once("Enabled separate cuda stream for MoE shared_experts")
|
||||
|
||||
# TODO(bnell): Hack for elastic_ep. Get rid of this
|
||||
def _set_moe_config(self, new_moe_config: FusedMoEConfig):
|
||||
self.moe_config = new_moe_config
|
||||
|
||||
@property
|
||||
def _disable_shared_experts_overlap(self) -> bool:
|
||||
# Disable shared expert overlap if:
|
||||
@@ -96,7 +93,7 @@ class SharedExperts:
|
||||
if self._disable_shared_experts_overlap:
|
||||
return SharedExpertsOrder.NO_OVERLAP
|
||||
|
||||
if self._quant_method.mk_can_overlap_shared_experts:
|
||||
if self._mk_can_overlap_shared_experts():
|
||||
return SharedExpertsOrder.MK_INTERNAL_OVERLAPPED
|
||||
|
||||
should_run_shared_in_aux_stream = (
|
||||
@@ -155,7 +152,7 @@ class SharedExperts:
|
||||
self._output[self._output_idx] = None
|
||||
return output
|
||||
|
||||
def apply(
|
||||
def forward(
|
||||
self,
|
||||
shared_experts_input: torch.Tensor,
|
||||
order: SharedExpertsOrder,
|
||||
|
||||
@@ -6,7 +6,6 @@ from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch.nn import Module
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
@@ -38,7 +37,7 @@ from vllm.platforms import current_platform
|
||||
from vllm.platforms.interface import CpuArchEnum
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -80,7 +79,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
def select_gemm_impl(
|
||||
self,
|
||||
prepare_finalize: FusedMoEPrepareAndFinalizeModular,
|
||||
layer: torch.nn.Module,
|
||||
layer: "RoutedExperts",
|
||||
) -> FusedMoEExpertsModular:
|
||||
raise ValueError(
|
||||
f"{self.__class__.__name__} uses the new modular kernel initialization "
|
||||
@@ -89,7 +88,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
layer: "RoutedExperts",
|
||||
num_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
@@ -156,7 +155,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
|
||||
def _setup_kernel(
|
||||
self,
|
||||
layer: Module,
|
||||
layer: "RoutedExperts",
|
||||
w13: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
) -> None:
|
||||
@@ -201,7 +200,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
def process_weights_after_loading(self, layer: "RoutedExperts") -> None:
|
||||
super().process_weights_after_loading(layer)
|
||||
|
||||
# Padding the weight for better performance on ROCm.
|
||||
|
||||
@@ -16,16 +16,14 @@ from vllm.model_executor.kernels.linear import (
|
||||
choose_mp_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEMethodBase,
|
||||
FusedMoEQuantConfig,
|
||||
FusedMoeWeightScaleSupported,
|
||||
RoutedExperts,
|
||||
SharedExperts,
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
|
||||
convert_to_wna16_moe_kernel_format,
|
||||
make_wna16_moe_kernel,
|
||||
|
||||
@@ -210,7 +210,7 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
)
|
||||
return None
|
||||
|
||||
def _add_fused_moe_to_target_scheme_map(self):
|
||||
def _add_fused_moe_to_target_scheme_map(self): # XXXXXXXXXXXXXXXXXXXXXX
|
||||
"""
|
||||
Helper function to update target_scheme_map
|
||||
since linear layers get fused into FusedMoE
|
||||
@@ -219,10 +219,10 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
"""
|
||||
if (
|
||||
"Linear" not in self.target_scheme_map
|
||||
or "FusedMoE" in self.target_scheme_map
|
||||
or "RoutedExperts" in self.target_scheme_map
|
||||
):
|
||||
return
|
||||
self.target_scheme_map["FusedMoE"] = self.target_scheme_map["Linear"]
|
||||
self.target_scheme_map["RoutedExperts"] = self.target_scheme_map["Linear"]
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config: dict[str, Any]) -> "CompressedTensorsConfig":
|
||||
|
||||
@@ -10,14 +10,12 @@ import torch
|
||||
|
||||
from vllm import envs
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEMethodBase,
|
||||
FusedMoEQuantConfig,
|
||||
RoutedExperts,
|
||||
SharedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
@@ -44,8 +42,9 @@ from vllm.model_executor.parameter import (
|
||||
)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
if current_platform.is_cuda():
|
||||
if has_humming() and current_platform.is_cuda():
|
||||
from humming.dtypes import DataType
|
||||
from humming.layer import HummingMethod
|
||||
from humming.schema import (
|
||||
|
||||
@@ -155,6 +155,14 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
# so can skip the padding in the forward before applying the moe method
|
||||
return self.mxfp4_backend == Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8
|
||||
|
||||
# TODO(bnell): move to MK/expert_class?
|
||||
@property
|
||||
def has_unpadded_output(self) -> bool:
|
||||
return self.mxfp4_backend in [
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
|
||||
]
|
||||
|
||||
def maybe_roundup_sizes(
|
||||
self,
|
||||
hidden_size: int,
|
||||
@@ -487,6 +495,14 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
# so can skip the padding in the forward before applying the moe method
|
||||
return self.mxfp4_backend == Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8
|
||||
|
||||
# TODO(bnell): move to MK/expert_class?
|
||||
@property
|
||||
def has_unpadded_output(self) -> bool:
|
||||
return self.mxfp4_backend in [
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
|
||||
]
|
||||
|
||||
def maybe_roundup_sizes(
|
||||
self,
|
||||
hidden_size: int,
|
||||
|
||||
@@ -234,7 +234,12 @@ def check_moe_marlin_supports_layer(layer: RoutedExperts, group_size: int) -> bo
|
||||
if current_platform.is_rocm():
|
||||
return False
|
||||
hidden_size = layer.hidden_size
|
||||
intermediate_size_per_partition = layer.intermediate_size_per_partition
|
||||
# Note: The layer has not performed rounding on intermediate_size's at this
|
||||
# point. Use the unpadded size which won't change.
|
||||
intermediate_size_per_partition = (
|
||||
layer.moe_config.intermediate_size_per_partition_unpadded
|
||||
)
|
||||
assert intermediate_size_per_partition is not None
|
||||
# apply_router_weight_on_input is not supported for moe marlin
|
||||
supports_router_weight = not layer.apply_router_weight_on_input
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import torch
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
USE_FP32_REDUCE_DEFAULT,
|
||||
get_marlin_input_dtype,
|
||||
@@ -286,7 +287,7 @@ def prepare_fp4_layer_for_marlin(
|
||||
|
||||
|
||||
def prepare_nvfp4_moe_layer_for_marlin(
|
||||
layer: torch.nn.Module,
|
||||
layer: RoutedExperts,
|
||||
w13: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
w13_scale_2: torch.Tensor,
|
||||
@@ -353,7 +354,7 @@ def prepare_nvfp4_moe_layer_for_marlin(
|
||||
|
||||
# WEIGHT SCALES
|
||||
# Permute scales
|
||||
def premute_scales(
|
||||
def permute_scales(
|
||||
scales: torch.Tensor, g_scales: torch.Tensor, name: str
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
scales = scales.to(param_dtype)
|
||||
@@ -388,8 +389,8 @@ def prepare_nvfp4_moe_layer_for_marlin(
|
||||
g_scales = g_scales / combined_scale_factor
|
||||
return scales, g_scales
|
||||
|
||||
w13_scale, w13_scale_2 = premute_scales(w13_scale, w13_scale_2, "w13")
|
||||
w2_scale, w2_scale_2 = premute_scales(w2_scale, w2_scale_2, "w2")
|
||||
w13_scale, w13_scale_2 = permute_scales(w13_scale, w13_scale_2, "w13")
|
||||
w2_scale, w2_scale_2 = permute_scales(w2_scale, w2_scale_2, "w2")
|
||||
|
||||
return w13, w13_scale, w13_scale_2, w2, w2_scale, w2_scale_2
|
||||
|
||||
|
||||
Reference in New Issue
Block a user