feat: extended EPLB support for Mistral Large 3 and additional MoE backends (#48355)

Signed-off-by: jdebache <jdebache@nvidia.com>
This commit is contained in:
Julien Debache
2026-08-07 05:32:52 -07:00
committed by GitHub
parent 8d9b52f7c2
commit ae934ba8a5
16 changed files with 500 additions and 96 deletions
@@ -15,6 +15,9 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
swizzle_blockscale,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
amax_for_moe_activation_quant,
)
if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe import RoutedExperts
@@ -129,12 +132,13 @@ def prepare_nvfp4_moe_layer_for_flashinfer_cutedsl(
and interleaves w13 gate/linear rows for gated activations. Non-gated
activations use a single w13 projection and keep its row order unchanged.
"""
from flashinfer.cute_dsl.utils import convert_sf_to_mma_layout
# Global scaling factors (same as other FlashInfer backends).
num_experts = w13.shape[0]
a13_scale = a13_scale.max().to(torch.float32).repeat(num_experts)
a2_scale = a2_scale.max().to(torch.float32).repeat(num_experts)
enable_eplb = layer.moe_config.moe_parallel_config.enable_eplb
a13_scale = amax_for_moe_activation_quant(a13_scale, enable_eplb).repeat(
num_experts
)
a2_scale = amax_for_moe_activation_quant(a2_scale, enable_eplb).repeat(num_experts)
if layer.activation.is_gated:
w13, w13_scale = reorder_w13_to_w31_for_flashinfer_cutedsl(
@@ -145,29 +149,8 @@ def prepare_nvfp4_moe_layer_for_flashinfer_cutedsl(
w13 = interleave_linear_and_gate(w13, group_size=64, dim=1)
w13_scale = interleave_linear_and_gate(w13_scale, group_size=64, dim=1)
# Convert w13 scale factors: linear → swizzled → MMA layout.
w13_scale = swizzle_blockscale(w13_scale)
E, M_padded, K_sf_padded = w13_scale.shape
w13_scale_flat = w13_scale.reshape(E * M_padded, K_sf_padded)
w13_scale = convert_sf_to_mma_layout(
w13_scale_flat,
m=M_padded,
k=K_sf_padded * 16,
num_groups=E,
sf_vec_size=16,
)
# Convert w2 scale factors: linear → swizzled → MMA layout.
w2_scale = swizzle_blockscale(w2_scale)
E, M_padded, K_sf_padded = w2_scale.shape
w2_scale_flat = w2_scale.reshape(E * M_padded, K_sf_padded)
w2_scale = convert_sf_to_mma_layout(
w2_scale_flat,
m=M_padded,
k=K_sf_padded * 16,
num_groups=E,
sf_vec_size=16,
)
return (
w13,
@@ -181,6 +164,31 @@ def prepare_nvfp4_moe_layer_for_flashinfer_cutedsl(
)
def nvfp4_swizzled_scale_to_cutedsl_mma_view(scale: torch.Tensor) -> torch.Tensor:
"""View a swizzled (E, M_padded, K_sf_padded) block-scale tensor in the
MMA layout expected by the CuteDSL MoE kernel.
The returned tensor aliases `scale`'s storage, so in-place updates of the
registered Parameter (weight reloads, EPLB rearrangement) are visible to
the kernel with no extra bookkeeping.
"""
from flashinfer.cute_dsl.utils import convert_sf_to_mma_layout
num_experts, m_padded, k_sf_padded = scale.shape
mma_view = convert_sf_to_mma_layout(
scale.reshape(num_experts * m_padded, k_sf_padded),
m=m_padded,
k=k_sf_padded * 16,
num_groups=num_experts,
sf_vec_size=16,
)
assert mma_view.data_ptr() == scale.data_ptr(), (
"convert_sf_to_mma_layout no longer returns a view of its input; "
"the quant config would go stale after weight updates."
)
return mma_view
def prepare_static_weights_for_trtllm_fp4_moe(
# args_dequant,
# args,
@@ -363,8 +371,13 @@ def prepare_nvfp4_moe_layer_for_fi_or_cutlass(
# For some FI kernels, the input scales are shared by all experts.
if is_global_sf_supported_for_nvfp4_backend(backend):
num_experts = w13.shape[0]
a13_scale = a13_scale.max().to(torch.float32).repeat(num_experts)
a2_scale = a2_scale.max().to(torch.float32).repeat(num_experts)
enable_eplb = layer.moe_config.moe_parallel_config.enable_eplb
a13_scale = amax_for_moe_activation_quant(a13_scale, enable_eplb).repeat(
num_experts
)
a2_scale = amax_for_moe_activation_quant(a2_scale, enable_eplb).repeat(
num_experts
)
else:
a13_scale = a13_scale.max(dim=1).values.to(torch.float32)
@@ -14,6 +14,7 @@ import vllm.envs as envs
from vllm import _custom_ops as ops
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.quant_utils import (
amax_for_moe_activation_quant,
get_fp8_min_max,
)
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
@@ -1437,6 +1438,7 @@ def process_fp8_weight_tensor_strategy_moe(
def process_fp8_input_tensor_strategy_moe(
w13_input_scale: torch.Tensor,
w2_input_scale: torch.Tensor,
enable_eplb: bool,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Process moe input scales for tensor-wise quantization strategy."""
@@ -1447,4 +1449,7 @@ def process_fp8_input_tensor_strategy_moe(
"for each layer."
)
return w13_input_scale.max(), w2_input_scale.max()
return (
amax_for_moe_activation_quant(w13_input_scale, enable_eplb),
amax_for_moe_activation_quant(w2_input_scale, enable_eplb),
)
@@ -62,6 +62,28 @@ def amax_for_moe_weight_quant(amax: torch.Tensor, moe_tp_size: int) -> torch.Ten
return amax
def amax_for_moe_activation_quant(
a_scale: torch.Tensor, enable_eplb: bool
) -> torch.Tensor:
"""Reduce a per-expert activation scale to one value shared by all experts.
Note: when EPLB is enabled and since this quantization scales get
folded into the per-expert dequantization alphas, we can only
ensure that the quant/dequant scales match by having a single
quantization scale shared across all ranks.
"""
a_max = a_scale.max().to(torch.float32)
if enable_eplb:
from vllm.distributed.parallel_state import get_ep_group
torch.distributed.all_reduce(
a_max,
op=torch.distributed.ReduceOp.MAX,
group=get_ep_group().device_group,
)
return a_max
def get_fp8_min_max() -> tuple[float, float]:
"""Get the min and max values for FP8 quantization."""
# Using the default value (240.0) from pytorch will cause accuracy