[feat] FlashInfer CuteDSL MegaMoE integration (#54049)
Signed-off-by: jdebache <jdebache@nvidia.com>
This commit is contained in:
@@ -31,6 +31,9 @@ from vllm.model_executor.layers.fused_moe.prepare_finalize.flashinfer_nvlink_one
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.flashinfer_nvlink_two_sided import ( # noqa: E501
|
||||
FlashInferNVLinkTwoSidedPrepareAndFinalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.passthrough import (
|
||||
PassThroughPrepareAndFinalize,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import (
|
||||
has_deep_ep,
|
||||
@@ -185,6 +188,10 @@ def maybe_make_prepare_finalize(
|
||||
all2all_manager: Any | None = None,
|
||||
input_dtype: torch.dtype | None = None,
|
||||
) -> FusedMoEPrepareAndFinalize | None:
|
||||
if moe.moe_parallel_config.use_passthrough_all2all:
|
||||
# The experts dispatch and combine themselves, in every EP topology.
|
||||
return PassThroughPrepareAndFinalize()
|
||||
|
||||
if not moe.moe_parallel_config.use_all2all_kernels:
|
||||
if not allow_new_interface:
|
||||
return None
|
||||
|
||||
@@ -1068,6 +1068,12 @@ class FusedMoEParallelConfig:
|
||||
and self.all2all_backend == "allgather_reducescatter"
|
||||
)
|
||||
|
||||
@property
|
||||
def use_passthrough_all2all(self):
|
||||
# Not gated on use_all2all_kernels: the experts dispatch and combine
|
||||
# internally in every EP topology, including TP-only.
|
||||
return self.all2all_backend == "passthrough"
|
||||
|
||||
@property
|
||||
def use_mori_kernels(self):
|
||||
return self.use_all2all_kernels and self.all2all_backend in (
|
||||
@@ -1530,6 +1536,10 @@ class FusedMoEConfig:
|
||||
def use_deepep_ll_kernels(self):
|
||||
return self.moe_parallel_config.use_deepep_ll_kernels
|
||||
|
||||
@property
|
||||
def use_passthrough_all2all(self):
|
||||
return self.moe_parallel_config.use_passthrough_all2all
|
||||
|
||||
@property
|
||||
def use_mori_kernels(self):
|
||||
return self.moe_parallel_config.use_mori_kernels
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""FlashInfer MoE-EP megakernels as a modular-kernel experts implementation.
|
||||
|
||||
The megakernel routes on vLLM's top-k output and then dispatches, computes and
|
||||
combines in one launch, so it is paired with a pass-through prepare/finalize.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.flashinfer_moe_ep import (
|
||||
FlashInferMoeEp,
|
||||
FlashInferMoeEpEpilogue,
|
||||
FlashInferMoeEpWeights,
|
||||
apply_topk_in_fc1,
|
||||
flashinfer_moe_ep_unsupported_reasons,
|
||||
supports_current_device,
|
||||
validate_flashinfer_moe_ep_layer,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceNoOP,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kMxfp4Static,
|
||||
kNvfp4Static,
|
||||
require_finite_positive_scales,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
|
||||
if TYPE_CHECKING:
|
||||
pass
|
||||
|
||||
# Layer parameters the megakernel reads. After ``process_weights_after_loading``
|
||||
# they alias the megakernel's own tensors, so EPLB permutes exactly what it uses.
|
||||
KERNEL_WEIGHT_NAMES = ("w13_weight", "w13_weight_scale", "w2_weight", "w2_weight_scale")
|
||||
|
||||
|
||||
_supported_weight_quant_schemes: frozenset[QuantKey] = frozenset(
|
||||
(kNvfp4Static, kMxfp4Static)
|
||||
)
|
||||
|
||||
_supported_activations: frozenset[MoEActivation] = frozenset((MoEActivation.SILU,))
|
||||
|
||||
|
||||
class FlashInferMoeEpExperts(mk.FusedMoEExpertsModular):
|
||||
def __init__(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
) -> None:
|
||||
super().__init__(moe_config, quant_config)
|
||||
self._adapter: FlashInferMoeEp | None = None
|
||||
|
||||
@staticmethod
|
||||
def is_supported_config(
|
||||
cls: type[mk.FusedMoEExperts],
|
||||
moe_config: FusedMoEConfig,
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
activation_format: mk.FusedMoEActivationFormat,
|
||||
) -> tuple[bool, str | None]:
|
||||
supported, reason = mk.FusedMoEExperts.is_supported_config(
|
||||
cls, moe_config, weight_key, activation_key, activation_format
|
||||
)
|
||||
if not supported:
|
||||
return False, reason
|
||||
reasons = flashinfer_moe_ep_unsupported_reasons(
|
||||
moe_config, weight_key, activation_key
|
||||
)
|
||||
if reasons:
|
||||
return False, f"kernel does not support {', '.join(reasons)}"
|
||||
return True, None
|
||||
|
||||
@staticmethod
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
return mk.FusedMoEActivationFormat.Standard
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
return supports_current_device()
|
||||
|
||||
@staticmethod
|
||||
def _supports_no_act_and_mul() -> bool:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
return weight_key in _supported_weight_quant_schemes
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
return activation in _supported_activations
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
|
||||
return (
|
||||
moe_parallel_config.use_ep
|
||||
and not moe_parallel_config.use_batched_activation_format
|
||||
)
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
return True
|
||||
|
||||
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
|
||||
return TopKWeightAndReduceNoOP()
|
||||
|
||||
def workspace_shapes(
|
||||
self,
|
||||
M: int,
|
||||
N: int,
|
||||
K: int,
|
||||
topk: int,
|
||||
global_num_experts: int,
|
||||
local_num_experts: int,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
activation: MoEActivation,
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
|
||||
return (0,), (0,), (M, K)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
"""Build the megakernel from the layer's canonical weights.
|
||||
|
||||
The oracle has already put the weights in kernel format; the per-expert
|
||||
weight global scales arrive through the quant config and become the
|
||||
epilogue alphas.
|
||||
"""
|
||||
validate_flashinfer_moe_ep_layer(layer) # type: ignore[arg-type]
|
||||
adapter = FlashInferMoeEp(
|
||||
self.moe_config,
|
||||
FlashInferMoeEpWeights(
|
||||
w13=layer.w13_weight,
|
||||
w2=layer.w2_weight,
|
||||
w13_scale=getattr(layer, "w13_weight_scale", None),
|
||||
w2_scale=getattr(layer, "w2_weight_scale", None),
|
||||
),
|
||||
epilogue_from_quant_config(self.quant_config),
|
||||
apply_topk_in_fc1=apply_topk_in_fc1(
|
||||
self.moe_config,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
),
|
||||
)
|
||||
adapter.warmup()
|
||||
for name, tensor in zip(KERNEL_WEIGHT_NAMES, adapter.kernel_weights()):
|
||||
replace_parameter(layer, name, tensor)
|
||||
self._adapter = adapter
|
||||
|
||||
def apply(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
a2_scale: torch.Tensor | None,
|
||||
workspace13: torch.Tensor,
|
||||
workspace2: torch.Tensor,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
) -> None:
|
||||
if self._adapter is None:
|
||||
raise RuntimeError("FlashInfer MoE-EP weights have not been processed")
|
||||
output.copy_(
|
||||
self._adapter(hidden_states, topk_ids.to(torch.int32), topk_weights)
|
||||
)
|
||||
|
||||
|
||||
def epilogue_from_quant_config(
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
) -> FlashInferMoeEpEpilogue:
|
||||
"""Per-expert epilogue constants from the canonical quant config.
|
||||
|
||||
The weight global scales (``g1_alphas``/``g2_alphas``) become the fc1/fc2
|
||||
alphas. Activations are quantized dynamically inside the megakernel, so the
|
||||
checkpoint's input scales drop out. MXFP4 checkpoints carry no global scales
|
||||
and keep the default epilogue. The alphas alias the layer's parameters, so
|
||||
EPLB permutations reach the kernel.
|
||||
"""
|
||||
fc1_alpha, fc2_alpha = quant_config.g1_alphas, quant_config.g2_alphas
|
||||
if fc1_alpha is None and fc2_alpha is None:
|
||||
return FlashInferMoeEpEpilogue()
|
||||
assert fc1_alpha is not None and fc2_alpha is not None
|
||||
require_finite_positive_scales("g1_alphas", fc1_alpha)
|
||||
require_finite_positive_scales("g2_alphas", fc2_alpha)
|
||||
if fc1_alpha.dim() != 1 or fc2_alpha.dim() != 1:
|
||||
raise ValueError("the megakernel needs one weight global scale per expert")
|
||||
return FlashInferMoeEpEpilogue(
|
||||
fc1_alpha=fc1_alpha.float().contiguous(),
|
||||
fc2_alpha=fc2_alpha.float().contiguous(),
|
||||
)
|
||||
@@ -0,0 +1,481 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, Literal, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.config.kernel import (
|
||||
FLASHINFER_MOE_EP_BACKENDS,
|
||||
FLASHINFER_MOE_EP_CUTEDSL,
|
||||
FLASHINFER_MOE_EP_DEEP_GEMM,
|
||||
)
|
||||
from vllm.distributed import get_ep_group
|
||||
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kMxfp4Static,
|
||||
kNvfp4Static,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import _import_deep_gemm
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
|
||||
|
||||
FlashInferMoeEpKernel = Literal["cutedsl", "deep_gemm"]
|
||||
FlashInferMoeEpWeightFormat = Literal["nvfp4", "mxfp4"]
|
||||
_WEIGHT_FORMATS: dict[QuantKey | None, FlashInferMoeEpWeightFormat] = {
|
||||
kNvfp4Static: "nvfp4",
|
||||
kMxfp4Static: "mxfp4",
|
||||
}
|
||||
|
||||
_SUPPORTED_CAPABILITIES = frozenset({(10, 0), (10, 3)})
|
||||
|
||||
|
||||
def supports_current_device() -> bool:
|
||||
capability = current_platform.get_device_capability()
|
||||
return capability is not None and (
|
||||
(capability.major, capability.minor) in _SUPPORTED_CAPABILITIES
|
||||
)
|
||||
|
||||
|
||||
_E2M1_LUT = (
|
||||
0.0,
|
||||
0.5,
|
||||
1.0,
|
||||
1.5,
|
||||
2.0,
|
||||
3.0,
|
||||
4.0,
|
||||
6.0,
|
||||
-0.0,
|
||||
-0.5,
|
||||
-1.0,
|
||||
-1.5,
|
||||
-2.0,
|
||||
-3.0,
|
||||
-4.0,
|
||||
-6.0,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FlashInferMoeEpBackendSpec:
|
||||
kernel: FlashInferMoeEpKernel
|
||||
weight_formats: frozenset[FlashInferMoeEpWeightFormat]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FlashInferMoeEpWeights:
|
||||
w13: torch.Tensor
|
||||
w2: torch.Tensor
|
||||
w13_scale: torch.Tensor | None = None
|
||||
w2_scale: torch.Tensor | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FlashInferMoeEpEpilogue:
|
||||
input_norm_const: float = 1.0
|
||||
fc1_alpha: torch.Tensor | None = None
|
||||
fc2_alpha: torch.Tensor | None = None
|
||||
fc1_norm_const: torch.Tensor | None = None
|
||||
|
||||
|
||||
_BACKEND_SPECS = {
|
||||
FLASHINFER_MOE_EP_CUTEDSL: FlashInferMoeEpBackendSpec(
|
||||
kernel="cutedsl",
|
||||
weight_formats=frozenset({"nvfp4", "mxfp4"}),
|
||||
),
|
||||
FLASHINFER_MOE_EP_DEEP_GEMM: FlashInferMoeEpBackendSpec(
|
||||
kernel="deep_gemm",
|
||||
weight_formats=frozenset({"mxfp4"}),
|
||||
),
|
||||
}
|
||||
|
||||
assert frozenset(_BACKEND_SPECS) == FLASHINFER_MOE_EP_BACKENDS
|
||||
|
||||
|
||||
def is_flashinfer_moe_ep_backend(moe_backend: str) -> bool:
|
||||
return moe_backend in _BACKEND_SPECS
|
||||
|
||||
|
||||
def flashinfer_moe_ep_backend_spec(
|
||||
moe_backend: str,
|
||||
) -> FlashInferMoeEpBackendSpec:
|
||||
try:
|
||||
return _BACKEND_SPECS[moe_backend]
|
||||
except KeyError:
|
||||
raise ValueError(
|
||||
f"{moe_backend!r} is not a FlashInfer MoE-EP backend; expected "
|
||||
f"one of {sorted(_BACKEND_SPECS)}"
|
||||
) from None
|
||||
|
||||
|
||||
def flashinfer_moe_ep_unsupported_reasons(
|
||||
moe: FusedMoEConfig,
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> tuple[str, ...]:
|
||||
"""Deployment constraints of the megakernel beyond the framework's checks.
|
||||
|
||||
``activation_key`` is ``None`` for MXFP4 checkpoints (the kernel quantizes
|
||||
bf16 activations itself) but marks a W4A16 recipe for NVFP4 weights.
|
||||
"""
|
||||
spec = flashinfer_moe_ep_backend_spec(moe.moe_backend)
|
||||
unsupported: list[str] = []
|
||||
weight_format = _WEIGHT_FORMATS.get(weight_key)
|
||||
if weight_format is None:
|
||||
unsupported.append(f"weight scheme {weight_key}")
|
||||
elif weight_format not in spec.weight_formats:
|
||||
unsupported.append(f"{weight_format.upper()} weights")
|
||||
if weight_key == kNvfp4Static and activation_key is None:
|
||||
unsupported.append("A16 activations with NVFP4 weights")
|
||||
if moe.skip_final_all_reduce:
|
||||
unsupported.append("skip_final_all_reduce")
|
||||
if moe.in_dtype != torch.bfloat16:
|
||||
unsupported.append(f"activation dtype {moe.in_dtype}")
|
||||
if moe.has_bias:
|
||||
unsupported.append("expert bias")
|
||||
if moe.swiglu_alpha is not None or moe.swiglu_beta is not None:
|
||||
unsupported.append("custom SwiGLU alpha or beta")
|
||||
if (
|
||||
spec.kernel == "deep_gemm"
|
||||
and moe.routing_method is not RoutingMethodType.DeepseekV4
|
||||
):
|
||||
unsupported.append(f"routing method {moe.routing_method.name}")
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
if vllm_config.weight_transfer_config is not None:
|
||||
unsupported.append("runtime weight transfer")
|
||||
if vllm_config.parallel_config.enable_dbo:
|
||||
unsupported.append("dual batch overlap")
|
||||
if spec.kernel == "deep_gemm" and vllm_config.parallel_config.enable_eplb:
|
||||
unsupported.append("EPLB")
|
||||
return tuple(unsupported)
|
||||
|
||||
|
||||
def validate_flashinfer_moe_ep_layer(layer: RoutedExperts) -> None:
|
||||
if layer.expert_map_manager.placement_strategy != "linear":
|
||||
raise ValueError(
|
||||
f"{layer.moe_config.moe_backend} requires contiguous linear "
|
||||
"expert placement"
|
||||
)
|
||||
|
||||
|
||||
def apply_topk_in_fc1(
|
||||
moe: FusedMoEConfig,
|
||||
*,
|
||||
apply_router_weight_on_input: bool = False,
|
||||
) -> bool:
|
||||
return (
|
||||
apply_router_weight_on_input
|
||||
or moe.routing_method is RoutingMethodType.DeepseekV4
|
||||
)
|
||||
|
||||
|
||||
def _dequant_mxfp4_ue8m0_gran32(
|
||||
packed: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
raw = packed.view(torch.uint8)
|
||||
lut = torch.tensor(_E2M1_LUT, dtype=torch.float32, device=raw.device)
|
||||
values = torch.empty(
|
||||
raw.shape[0],
|
||||
raw.shape[1] * 2,
|
||||
dtype=torch.float32,
|
||||
device=raw.device,
|
||||
)
|
||||
values[:, ::2] = lut[(raw & 0x0F).to(torch.int64)]
|
||||
values[:, 1::2] = lut[(raw >> 4).to(torch.int64)]
|
||||
encoded_scale = (
|
||||
scale
|
||||
if scale.dtype == torch.float8_e8m0fnu
|
||||
else scale.view(torch.float8_e8m0fnu)
|
||||
)
|
||||
decoded_scale = encoded_scale.float()
|
||||
return (values * decoded_scale.repeat_interleave(32, dim=-1)).to(torch.bfloat16)
|
||||
|
||||
|
||||
def _dequant_mxfp4_experts(
|
||||
weight: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
num_experts, output_size, packed_input_size = weight.shape
|
||||
result = torch.empty(
|
||||
num_experts,
|
||||
output_size,
|
||||
packed_input_size * 2,
|
||||
dtype=torch.bfloat16,
|
||||
device=weight.device,
|
||||
)
|
||||
for expert_id in range(num_experts):
|
||||
result[expert_id] = _dequant_mxfp4_ue8m0_gran32(
|
||||
weight[expert_id], scale[expert_id]
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def mxfp4_moe_ep_weights(
|
||||
moe_backend: str,
|
||||
w13: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
) -> FlashInferMoeEpWeights:
|
||||
spec = flashinfer_moe_ep_backend_spec(moe_backend)
|
||||
if spec.kernel == "deep_gemm":
|
||||
return FlashInferMoeEpWeights(w13, w2, w13_scale, w2_scale)
|
||||
return FlashInferMoeEpWeights(
|
||||
w13=_dequant_mxfp4_experts(w13, w13_scale),
|
||||
w2=_dequant_mxfp4_experts(w2, w2_scale),
|
||||
)
|
||||
|
||||
|
||||
class _FlashInferMoeEpApi:
|
||||
def __init__(self) -> None:
|
||||
try:
|
||||
from flashinfer.moe_ep import (
|
||||
BootstrapConfig,
|
||||
DeepGemmMegaMoeConfig,
|
||||
FleetParams,
|
||||
MegaConfig,
|
||||
MoEEpMegaLayer,
|
||||
MoEEpTensors,
|
||||
MoEWeightPack,
|
||||
Nvfp4CutedslMegaMoeConfig,
|
||||
)
|
||||
except ImportError as err:
|
||||
raise RuntimeError(
|
||||
"FlashInfer MoE-EP requires the public API shipped with "
|
||||
"FlashInfer 0.7.0 or newer"
|
||||
) from err
|
||||
|
||||
self.BootstrapConfig = BootstrapConfig
|
||||
self.DeepGemmMegaMoeConfig = DeepGemmMegaMoeConfig
|
||||
self.FleetParams = FleetParams
|
||||
self.MegaConfig = MegaConfig
|
||||
self.MoEEpMegaLayer = MoEEpMegaLayer
|
||||
self.MoEEpTensors = MoEEpTensors
|
||||
self.MoEWeightPack = MoEWeightPack
|
||||
self.Nvfp4CutedslMegaMoeConfig = Nvfp4CutedslMegaMoeConfig
|
||||
|
||||
|
||||
def _load_flashinfer_moe_ep_api() -> _FlashInferMoeEpApi:
|
||||
return _FlashInferMoeEpApi()
|
||||
|
||||
|
||||
def _expose_deep_gemm_to_flashinfer() -> None:
|
||||
deep_gemm = _import_deep_gemm()
|
||||
if deep_gemm is None:
|
||||
raise RuntimeError(
|
||||
"FlashInfer MoE-EP DeepGEMM requires a usable DeepGEMM installation"
|
||||
)
|
||||
sys.modules.setdefault("deep_gemm", deep_gemm)
|
||||
|
||||
|
||||
class FlashInferMoeEp:
|
||||
def __init__(
|
||||
self,
|
||||
moe: FusedMoEConfig,
|
||||
weights: FlashInferMoeEpWeights,
|
||||
epilogue: FlashInferMoeEpEpilogue | None = None,
|
||||
*,
|
||||
apply_topk_in_fc1: bool,
|
||||
) -> None:
|
||||
spec = flashinfer_moe_ep_backend_spec(moe.moe_backend)
|
||||
if spec.kernel == "deep_gemm":
|
||||
_expose_deep_gemm_to_flashinfer()
|
||||
api = _load_flashinfer_moe_ep_api()
|
||||
if epilogue is None:
|
||||
epilogue = FlashInferMoeEpEpilogue()
|
||||
ep_group = get_ep_group()
|
||||
bootstrap = api.BootstrapConfig(
|
||||
world_size=ep_group.world_size,
|
||||
rank=ep_group.rank_in_group,
|
||||
process_group=ep_group.device_group,
|
||||
device=torch.accelerator.current_device_index(),
|
||||
)
|
||||
fleet_params = api.FleetParams(
|
||||
num_experts=moe.num_experts,
|
||||
max_tokens_per_rank=moe.max_num_tokens,
|
||||
token_hidden_size=moe.hidden_dim,
|
||||
)
|
||||
weight_pack = api.MoEWeightPack(
|
||||
w13=weights.w13,
|
||||
w2=weights.w2,
|
||||
w13_scale=weights.w13_scale,
|
||||
w2_scale=weights.w2_scale,
|
||||
)
|
||||
if spec.kernel == "cutedsl":
|
||||
megakernel = api.Nvfp4CutedslMegaMoeConfig(
|
||||
intermediate_size=moe.intermediate_size,
|
||||
top_k=moe.experts_per_token,
|
||||
gate_up_clamp=moe.swiglu_limit,
|
||||
fast_math=True,
|
||||
apply_topk_in_fc1=apply_topk_in_fc1,
|
||||
enable_in_kernel_fc2_reduce=False,
|
||||
combine_dtype="bf16",
|
||||
input_norm_const=epilogue.input_norm_const,
|
||||
fc1_alpha=None,
|
||||
fc2_alpha=None,
|
||||
fc1_norm_const=None,
|
||||
knobs=None,
|
||||
)
|
||||
else:
|
||||
megakernel = api.DeepGemmMegaMoeConfig(
|
||||
intermediate_size=moe.intermediate_size,
|
||||
top_k=moe.experts_per_token,
|
||||
activation_clamp=moe.swiglu_limit,
|
||||
fast_math=True,
|
||||
)
|
||||
backend = api.MegaConfig(
|
||||
megakernel=megakernel,
|
||||
quantize_input=True,
|
||||
preprocess_weights=True,
|
||||
)
|
||||
|
||||
self._moe_ep_tensors_cls = api.MoEEpTensors
|
||||
self._mega_layer: Any | None = api.MoEEpMegaLayer(
|
||||
bootstrap,
|
||||
fleet_params,
|
||||
weight_pack,
|
||||
backend,
|
||||
)
|
||||
self._device = weights.w13.device
|
||||
self._hidden_size = moe.hidden_dim
|
||||
self._max_num_tokens = moe.max_num_tokens
|
||||
self._num_experts = moe.num_experts
|
||||
self._top_k = moe.experts_per_token
|
||||
self._epilogue: FlashInferMoeEpEpilogue | None = epilogue
|
||||
self._return_workspace_view = moe.routing_method is RoutingMethodType.DeepseekV4
|
||||
|
||||
@property
|
||||
def can_overlap_shared_experts(self) -> bool:
|
||||
return False
|
||||
|
||||
@property
|
||||
def output_is_reduced(self) -> bool:
|
||||
return True
|
||||
|
||||
@property
|
||||
def topk_indices_dtype(self) -> torch.dtype:
|
||||
return torch.int32
|
||||
|
||||
@property
|
||||
def is_monolithic(self) -> bool:
|
||||
return False
|
||||
|
||||
def _tensors(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
) -> Any:
|
||||
epilogue = cast(FlashInferMoeEpEpilogue, self._epilogue)
|
||||
return self._moe_ep_tensors_cls(
|
||||
hidden_states=hidden_states,
|
||||
topk_ids=topk_ids,
|
||||
topk_weights=topk_weights,
|
||||
fc1_alpha=epilogue.fc1_alpha,
|
||||
fc2_alpha=epilogue.fc2_alpha,
|
||||
fc1_norm_const=epilogue.fc1_norm_const,
|
||||
)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
if self._mega_layer is None:
|
||||
raise RuntimeError("FlashInfer MoE-EP layer was destroyed")
|
||||
if hidden_states.shape[0] > self._max_num_tokens:
|
||||
raise ValueError(
|
||||
f"FlashInfer MoE-EP got {hidden_states.shape[0]} tokens, but "
|
||||
f"the workspace supports at most {self._max_num_tokens}"
|
||||
)
|
||||
tensors = self._tensors(hidden_states, topk_ids, topk_weights)
|
||||
if self._return_workspace_view and getattr(
|
||||
self._mega_layer, "supports_output_view", False
|
||||
):
|
||||
return self._mega_layer.forward(tensors, return_workspace_view=True)
|
||||
return self._mega_layer(tensors)
|
||||
|
||||
@torch.inference_mode()
|
||||
def warmup(self) -> None:
|
||||
if self._mega_layer is None:
|
||||
return
|
||||
hidden_states = torch.zeros(
|
||||
1,
|
||||
self._hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=self._device,
|
||||
)
|
||||
topk_ids = (
|
||||
torch.arange(self._top_k, dtype=torch.int32, device=self._device)
|
||||
.mul_(self._num_experts // self._top_k)
|
||||
.view(1, self._top_k)
|
||||
)
|
||||
topk_weights = torch.full(
|
||||
(1, self._top_k),
|
||||
1.0 / self._top_k,
|
||||
dtype=torch.float32,
|
||||
device=self._device,
|
||||
)
|
||||
self._mega_layer.warmup(self._tensors(hidden_states, topk_ids, topk_weights))
|
||||
|
||||
def kernel_weights(
|
||||
self,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Kernel-resident fc1 weight, fc1 scales, fc2 weight, fc2 scales.
|
||||
|
||||
Contiguous per-expert byte tensors that alias the memory the megakernel
|
||||
reads, so a caller may permute experts in place (EPLB).
|
||||
"""
|
||||
if self._mega_layer is None:
|
||||
raise RuntimeError("FlashInfer MoE-EP layer was destroyed")
|
||||
(fc1_weight, fc1_sf), (fc2_weight, fc2_sf) = self._mega_layer._transformed
|
||||
return (
|
||||
_expert_major_bytes(fc1_weight),
|
||||
_expert_major_bytes(fc1_sf),
|
||||
_expert_major_bytes(fc2_weight),
|
||||
_expert_major_bytes(fc2_sf),
|
||||
)
|
||||
|
||||
def expert_weight_views(self) -> tuple[torch.Tensor, ...]:
|
||||
"""Every per-expert tensor the kernel reads: weights plus epilogue."""
|
||||
epilogue = cast(FlashInferMoeEpEpilogue, self._epilogue)
|
||||
vectors = (epilogue.fc1_alpha, epilogue.fc2_alpha, epilogue.fc1_norm_const)
|
||||
return (*self.kernel_weights(), *(v for v in vectors if v is not None))
|
||||
|
||||
def destroy(self) -> None:
|
||||
mega_layer = self._mega_layer
|
||||
if mega_layer is None:
|
||||
return
|
||||
mega_layer.destroy()
|
||||
self._mega_layer = None
|
||||
self._epilogue = None
|
||||
|
||||
def __del__(self) -> None:
|
||||
with contextlib.suppress(Exception):
|
||||
self.destroy()
|
||||
|
||||
|
||||
def _expert_major_bytes(tensor: torch.Tensor) -> torch.Tensor:
|
||||
"""Contiguous storage of a per-expert kernel tensor, 1-byte dtypes as uint8.
|
||||
|
||||
The CuTeDSL kernel keeps (E, K, N) transpose views over (E, N, K) storage.
|
||||
"""
|
||||
if not tensor.is_contiguous():
|
||||
tensor = tensor.transpose(1, 2)
|
||||
if not tensor.is_contiguous():
|
||||
raise ValueError(f"expert tensor is not expert-major: {tensor.stride()}")
|
||||
return tensor.view(torch.uint8) if tensor.element_size() == 1 else tensor
|
||||
@@ -8,7 +8,11 @@ import torch
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm import envs
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.config.kernel import MoEBackend
|
||||
from vllm.config.kernel import (
|
||||
FLASHINFER_MOE_EP_CUTEDSL,
|
||||
FLASHINFER_MOE_EP_DEEP_GEMM,
|
||||
MoEBackend,
|
||||
)
|
||||
from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
@@ -145,6 +149,9 @@ class Mxfp4MoeBackend(Enum):
|
||||
EMULATION = "EMULATION"
|
||||
# Humming
|
||||
HUMMING = "HUMMING"
|
||||
# FlashInfer MoE-EP megakernels: routing outside, dispatch/GEMMs/combine fused.
|
||||
FLASHINFER_MOE_EP_CUTEDSL = "FLASHINFER_MOE_EP_CUTEDSL"
|
||||
FLASHINFER_MOE_EP_DEEP_GEMM = "FLASHINFER_MOE_EP_DEEP_GEMM"
|
||||
|
||||
|
||||
# Backends that share the same TRTLLM weight format
|
||||
@@ -165,6 +172,13 @@ B12X_BACKENDS = (
|
||||
Mxfp4MoeBackend.B12X_MXFP4_BF16,
|
||||
)
|
||||
|
||||
# Megakernels consume the checkpoint's MXFP4 tensors through the shared
|
||||
# FlashInfer MoE-EP adapter; vLLM does not convert their weight layout.
|
||||
FLASHINFER_MOE_EP_MXFP4_BACKENDS = (
|
||||
Mxfp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL,
|
||||
Mxfp4MoeBackend.FLASHINFER_MOE_EP_DEEP_GEMM,
|
||||
)
|
||||
|
||||
|
||||
def backend_to_kernel_cls(
|
||||
backend: Mxfp4MoeBackend,
|
||||
@@ -291,6 +305,13 @@ def backend_to_kernel_cls(
|
||||
|
||||
return [OCP_MXQuantizationEmulationTritonExperts]
|
||||
|
||||
elif backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
|
||||
from vllm.model_executor.layers.fused_moe.experts.flashinfer_moe_ep import ( # noqa: E501
|
||||
FlashInferMoeEpExperts,
|
||||
)
|
||||
|
||||
return [FlashInferMoeEpExperts]
|
||||
|
||||
else:
|
||||
raise ValueError(f"Unknown MXFP4 MoE backend: {backend.value}")
|
||||
|
||||
@@ -314,6 +335,10 @@ def map_mxfp4_backend(runner_backend: MoEBackend) -> list[Mxfp4MoeBackend]:
|
||||
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
|
||||
],
|
||||
"flashinfer_cutlass_afp8": [Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8],
|
||||
"flashinfer_moe_ep_cutedsl": [Mxfp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL],
|
||||
"flashinfer_moe_ep_mega_deep_gemm": [
|
||||
Mxfp4MoeBackend.FLASHINFER_MOE_EP_DEEP_GEMM
|
||||
],
|
||||
"triton": [Mxfp4MoeBackend.TRITON],
|
||||
"triton_unfused": [Mxfp4MoeBackend.TRITON_UNFUSED],
|
||||
"humming": [Mxfp4MoeBackend.HUMMING],
|
||||
@@ -735,6 +760,9 @@ def mxfp4_round_up_hidden_size_and_intermediate_size(
|
||||
activation: MoEActivation | None = None,
|
||||
) -> tuple[int, int]:
|
||||
"""Round up hidden_size and intermediate_size based on backend requirements."""
|
||||
if backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
|
||||
# The megakernel takes the model's dimensions and validates them itself.
|
||||
return hidden_size, intermediate_size
|
||||
if backend in B12X_BACKENDS:
|
||||
# b12x plans for the exact model dimensions. B12xExperts validates the
|
||||
# required MXFP4 block alignment before selecting the backend.
|
||||
@@ -1827,6 +1855,29 @@ def convert_weight_to_mxfp4_moe_kernel_format(
|
||||
w2_bias=w2_bias,
|
||||
_cache_permute_indices=_cache_permute_indices,
|
||||
)
|
||||
elif mxfp4_backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
|
||||
from vllm.model_executor.layers.fused_moe.flashinfer_moe_ep import (
|
||||
mxfp4_moe_ep_weights,
|
||||
)
|
||||
|
||||
# DeepGEMM consumes MXFP4 directly; CuTeDSL gets bf16 and requantizes.
|
||||
weights = mxfp4_moe_ep_weights(
|
||||
FLASHINFER_MOE_EP_CUTEDSL
|
||||
if mxfp4_backend == Mxfp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL
|
||||
else FLASHINFER_MOE_EP_DEEP_GEMM,
|
||||
w13_weight,
|
||||
w2_weight,
|
||||
w13_weight_scale,
|
||||
w2_weight_scale,
|
||||
)
|
||||
return (
|
||||
weights.w13,
|
||||
weights.w2,
|
||||
weights.w13_scale,
|
||||
weights.w2_scale,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
)
|
||||
elif mxfp4_backend == Mxfp4MoeBackend.CPU:
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
prepare_mxfp4_moe_layer_for_cpu,
|
||||
@@ -1874,6 +1925,9 @@ def make_mxfp4_moe_quant_config(
|
||||
layer: "RoutedExperts | None" = None,
|
||||
) -> FusedMoEQuantConfig | None:
|
||||
"""Create a FusedMoEQuantConfig for the given MXFP4 backend."""
|
||||
if mxfp4_backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
|
||||
# The megakernel quantizes activations itself and owns its scales.
|
||||
return FusedMoEQuantConfig.make("nvfp4", weight_dtype="mxfp4")
|
||||
if mxfp4_backend == Mxfp4MoeBackend.B12X_MXFP4_MXFP8:
|
||||
return mxfp4_mxfp8_moe_quant_config(
|
||||
w1_bias=w1_bias,
|
||||
|
||||
@@ -50,6 +50,7 @@ class NvFp4MoeBackend(Enum):
|
||||
MARLIN = "MARLIN"
|
||||
HUMMING = "HUMMING"
|
||||
EMULATION = "EMULATION"
|
||||
FLASHINFER_MOE_EP_CUTEDSL = "FLASHINFER_MOE_EP_CUTEDSL"
|
||||
|
||||
|
||||
FLASHINFER_NVFP4_MOE_BACKENDS = [
|
||||
@@ -148,6 +149,12 @@ def backend_to_kernel_cls(
|
||||
)
|
||||
|
||||
return [Nvfp4QuantizationEmulationTritonExperts]
|
||||
elif backend == NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL:
|
||||
from vllm.model_executor.layers.fused_moe.experts.flashinfer_moe_ep import ( # noqa: E501
|
||||
FlashInferMoeEpExperts,
|
||||
)
|
||||
|
||||
return [FlashInferMoeEpExperts]
|
||||
else:
|
||||
raise ValueError(f"Unknown NvFP4 MoE backend: {backend.value}")
|
||||
|
||||
@@ -160,6 +167,7 @@ def map_nvfp4_backend(runner_backend: MoEBackend) -> NvFp4MoeBackend:
|
||||
"flashinfer_trtllm": NvFp4MoeBackend.FLASHINFER_TRTLLM,
|
||||
"flashinfer_cutlass": NvFp4MoeBackend.FLASHINFER_CUTLASS,
|
||||
"flashinfer_cutedsl": NvFp4MoeBackend.FLASHINFER_CUTEDSL,
|
||||
"flashinfer_moe_ep_cutedsl": NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL,
|
||||
"flashinfer_b12x": NvFp4MoeBackend.FLASHINFER_B12X,
|
||||
"marlin": NvFp4MoeBackend.MARLIN,
|
||||
"humming": NvFp4MoeBackend.HUMMING,
|
||||
@@ -207,6 +215,7 @@ def select_nvfp4_moe_backend(
|
||||
NvFp4MoeBackend.FLASHINFER_TRTLLM,
|
||||
NvFp4MoeBackend.FLASHINFER_CUTLASS,
|
||||
NvFp4MoeBackend.FLASHINFER_CUTEDSL,
|
||||
NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL,
|
||||
NvFp4MoeBackend.VLLM_CUTLASS,
|
||||
NvFp4MoeBackend.MARLIN,
|
||||
NvFp4MoeBackend.EMULATION,
|
||||
@@ -281,8 +290,9 @@ def select_nvfp4_moe_backend(
|
||||
f"Model sets swiglu_limit={config.swiglu_limit}, but the "
|
||||
f"explicitly requested moe_backend={runner_backend!r} does "
|
||||
f"not apply the SwiGLU clamp. Use 'flashinfer_trtllm', "
|
||||
f"'flashinfer_cutlass', 'flashinfer_cutedsl', 'cutlass', "
|
||||
f"'b12x', 'marlin', or 'humming' instead."
|
||||
f"'flashinfer_cutlass', 'flashinfer_cutedsl', "
|
||||
f"'flashinfer_moe_ep_cutedsl', 'cutlass', 'b12x', 'marlin', "
|
||||
f"or 'humming' instead."
|
||||
)
|
||||
return _return_or_raise(
|
||||
requested_backend, config, weight_key, activation_key, activation_format
|
||||
@@ -491,6 +501,9 @@ def convert_to_nvfp4_moe_kernel_format(
|
||||
# for other experts - other selection strategies may be used.
|
||||
a13_scale = 1.0 / a13_scale.max().to(torch.float32)
|
||||
a2_scale = 1.0 / a2_scale.max().to(torch.float32)
|
||||
elif nvfp4_backend == NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL:
|
||||
# The megakernel transforms the checkpoint tensors itself at init.
|
||||
pass
|
||||
else:
|
||||
raise ValueError(f"Unknown NvFp4 backend for MoE: {nvfp4_backend}")
|
||||
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Prepare/finalize stages that do no expert-parallel communication.
|
||||
|
||||
Selected by ``all2all_backend="passthrough"``: the experts implementation
|
||||
dispatches, combines and reduces itself, so nothing is left for these stages.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.no_dp_ep import (
|
||||
MoEPrepareAndFinalizeNoDPEPModular,
|
||||
)
|
||||
|
||||
|
||||
class PassThroughPrepareAndFinalize(MoEPrepareAndFinalizeNoDPEPModular):
|
||||
"""Hands activations and routing straight to experts that own their EP
|
||||
communication."""
|
||||
|
||||
def output_is_reduced(self) -> bool:
|
||||
return True
|
||||
|
||||
def supports_deferred_moe_finalize(self) -> bool:
|
||||
# The experts apply the top-k weights and combine internally; there
|
||||
# is no finalize left for a consumer to take over.
|
||||
return False
|
||||
|
||||
def prepare(
|
||||
self,
|
||||
a1: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
defer_input_quant: bool = False,
|
||||
) -> mk.PrepareResultType:
|
||||
return a1, None, None, None, None
|
||||
+32
-5
@@ -24,10 +24,15 @@ from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
|
||||
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import (
|
||||
XPUExpertsMxFp4,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.flashinfer_moe_ep import (
|
||||
is_flashinfer_moe_ep_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
B12X_BACKENDS,
|
||||
FLASHINFER_MOE_EP_MXFP4_BACKENDS,
|
||||
Mxfp4MoeBackend,
|
||||
convert_weight_to_mxfp4_moe_kernel_format,
|
||||
make_mxfp4_moe_kernel,
|
||||
make_mxfp4_moe_quant_config,
|
||||
select_mxfp4_moe_backend,
|
||||
@@ -38,7 +43,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tenso
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
|
||||
prepare_moe_fp4_layer_for_marlin,
|
||||
)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -49,12 +54,14 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
super().__init__(moe)
|
||||
self.group_size = 32
|
||||
self.mxfp4_backend = Mxfp4MoeBackend.MARLIN
|
||||
self.experts_cls: type[mk.FusedMoEExperts] | None = None
|
||||
|
||||
# Backend selection must match the weight preparation below: CUTLASS
|
||||
# swizzles scales, b12x and XPU consume checkpoint packing, and Marlin
|
||||
# repacks weights and scales.
|
||||
# swizzles scales, b12x and XPU consume checkpoint packing, Marlin
|
||||
# repacks weights and scales, and the megakernels take the checkpoint
|
||||
# tensors as they are.
|
||||
self.use_cutlass_mxfp4 = CutlassExpertsMxfp4._supports_current_device()
|
||||
self.experts_cls: type[mk.FusedMoEExperts]
|
||||
if moe.moe_backend == "b12x":
|
||||
if moe.moe_backend == "b12x" or is_flashinfer_moe_ep_backend(moe.moe_backend):
|
||||
self.mxfp4_backend, experts_cls = select_mxfp4_moe_backend(moe)
|
||||
assert experts_cls is not None
|
||||
self.experts_cls = experts_cls
|
||||
@@ -70,6 +77,10 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
logger.info_once("Using MarlinExperts for MXFP4 MoE")
|
||||
self.experts_cls = MarlinExperts
|
||||
|
||||
@property
|
||||
def supports_eplb(self) -> bool:
|
||||
return self.mxfp4_backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
@@ -198,6 +209,21 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
layer.w2_weight_scale = torch.nn.Parameter(
|
||||
torch.stack(swizzled_w2), requires_grad=False
|
||||
)
|
||||
elif self.mxfp4_backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
|
||||
w13, w2, w13_scale, w2_scale, _, _ = (
|
||||
convert_weight_to_mxfp4_moe_kernel_format(
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
layer=layer,
|
||||
w13_weight=layer.w13_weight,
|
||||
w2_weight=layer.w2_weight,
|
||||
w13_weight_scale=layer.w13_weight_scale,
|
||||
w2_weight_scale=layer.w2_weight_scale,
|
||||
)
|
||||
)
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
replace_parameter(layer, "w2_weight", w2)
|
||||
replace_parameter(layer, "w13_weight_scale", w13_scale)
|
||||
replace_parameter(layer, "w2_weight_scale", w2_scale)
|
||||
elif self.mxfp4_backend in B12X_BACKENDS or current_platform.is_xpu():
|
||||
pass
|
||||
else:
|
||||
@@ -211,6 +237,7 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
if self.moe_quant_config is not None:
|
||||
assert self.experts_cls is not None
|
||||
self.moe_kernel = make_mxfp4_moe_kernel(
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
|
||||
+5
-3
@@ -38,6 +38,7 @@ _EPLB_SUPPORTED_NVFP4_BACKENDS = frozenset(
|
||||
NvFp4MoeBackend.FLASHINFER_CUTEDSL,
|
||||
NvFp4MoeBackend.FLASHINFER_CUTEDSL_BATCHED,
|
||||
NvFp4MoeBackend.FLASHINFER_TRTLLM,
|
||||
NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -61,7 +62,6 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
weight_key=kNvfp4Static,
|
||||
activation_key=None if use_a16 else kNvfp4Dynamic,
|
||||
)
|
||||
|
||||
self.use_global_sf = is_global_sf_supported_for_nvfp4_backend(
|
||||
self.nvfp4_backend
|
||||
)
|
||||
@@ -155,7 +155,8 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
set_weight_attrs(w13_weight_scale_2, extra_weight_attrs)
|
||||
|
||||
w2_weight_scale_2 = torch.nn.Parameter(
|
||||
torch.empty(num_experts, dtype=torch.float32), requires_grad=False
|
||||
torch.empty(num_experts, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_weight_global_scale", w2_weight_scale_2)
|
||||
extra_weight_attrs.update(
|
||||
@@ -175,7 +176,8 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
set_weight_attrs(w13_input_scale, extra_weight_attrs)
|
||||
|
||||
w2_input_scale = torch.nn.Parameter(
|
||||
torch.empty(num_experts, dtype=torch.float32), requires_grad=False
|
||||
torch.empty(num_experts, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_input_global_scale", w2_input_scale)
|
||||
extra_weight_attrs.update(
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.config.kernel import FLASHINFER_MOE_EP_BACKENDS
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoeWeightScaleSupported,
|
||||
@@ -53,6 +54,11 @@ class INCMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
|
||||
def __init__(self, moe) -> None:
|
||||
super().__init__(moe)
|
||||
if moe.moe_backend in FLASHINFER_MOE_EP_BACKENDS:
|
||||
raise ValueError(
|
||||
f"moe_backend={moe.moe_backend!r} is not supported for AutoRound "
|
||||
"(INC) MXFP4 MoE checkpoints"
|
||||
)
|
||||
self.group_size = 32
|
||||
# Backend selection must stay consistent with the weight preparation in
|
||||
# process_weights_after_loading / get_fused_moe_quant_config, which use
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
@@ -783,7 +784,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
def process_weights_after_loading(self, layer):
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
if self.mxfp4_backend == Mxfp4MoeBackend.NONE:
|
||||
return
|
||||
|
||||
|
||||
@@ -25,6 +25,11 @@ INT4_DTYPE = scalar_types.uint4b8
|
||||
INT8_DTYPE = scalar_types.uint8b128
|
||||
|
||||
|
||||
def require_finite_positive_scales(name: str, value: torch.Tensor) -> None:
|
||||
if not torch.isfinite(value).all() or not (value > 0).all():
|
||||
raise ValueError(f"{name} must contain finite positive scales")
|
||||
|
||||
|
||||
def _dtype_abbr(dtype: torch.dtype | ScalarType) -> str:
|
||||
"""Return a stable short name for torch and ScalarType dtypes."""
|
||||
if isinstance(dtype, ScalarType):
|
||||
|
||||
Reference in New Issue
Block a user