feat: extended EPLB support for Mistral Large 3 and additional MoE backends (#48355)
Signed-off-by: jdebache <jdebache@nvidia.com>
This commit is contained in:
+16
-2
@@ -15,6 +15,7 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
|
||||
NvFp4MoeBackend,
|
||||
convert_to_nvfp4_moe_kernel_format,
|
||||
is_global_sf_supported_for_nvfp4_backend,
|
||||
make_nvfp4_moe_kernel,
|
||||
@@ -30,6 +31,15 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
|
||||
|
||||
# NVFP4 backends that have been verified to support EPLB for this quantization recipe.
|
||||
_EPLB_SUPPORTED_NVFP4_BACKENDS = frozenset(
|
||||
{
|
||||
NvFp4MoeBackend.FLASHINFER_CUTEDSL,
|
||||
NvFp4MoeBackend.FLASHINFER_CUTEDSL_BATCHED,
|
||||
NvFp4MoeBackend.FLASHINFER_TRTLLM,
|
||||
}
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -54,6 +64,10 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
self.nvfp4_backend
|
||||
)
|
||||
|
||||
@property
|
||||
def supports_eplb(self) -> bool:
|
||||
return self.nvfp4_backend in _EPLB_SUPPORTED_NVFP4_BACKENDS
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
@@ -223,8 +237,8 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
replace_parameter(layer, "w13_weight_scale", w13_scale)
|
||||
replace_parameter(layer, "w2_weight", w2)
|
||||
replace_parameter(layer, "w2_weight_scale", w2_scale)
|
||||
layer.w13_weight_scale_2 = w13_scale_2
|
||||
layer.w2_weight_scale_2 = w2_scale_2
|
||||
replace_parameter(layer, "w13_weight_scale_2", w13_scale_2)
|
||||
replace_parameter(layer, "w2_weight_scale_2", w2_scale_2)
|
||||
layer.w13_input_scale = a13_scale
|
||||
layer.w2_input_scale = a2_scale
|
||||
|
||||
|
||||
+3
-1
@@ -290,7 +290,9 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
assert self.input_quant.strategy == QuantizationStrategy.TENSOR
|
||||
assert w13_input_scale is not None and w2_input_scale is not None
|
||||
w13_input_scale, w2_input_scale = process_fp8_input_tensor_strategy_moe(
|
||||
w13_input_scale, w2_input_scale
|
||||
w13_input_scale,
|
||||
w2_input_scale,
|
||||
layer.moe_config.moe_parallel_config.enable_eplb,
|
||||
)
|
||||
replace_parameter(layer, "w13_input_scale", w13_input_scale)
|
||||
replace_parameter(layer, "w2_input_scale", w2_input_scale)
|
||||
|
||||
@@ -709,7 +709,9 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
assert not self.block_quant
|
||||
assert w13_input_scale is not None and w2_input_scale is not None
|
||||
w13_input_scale, w2_input_scale = process_fp8_input_tensor_strategy_moe(
|
||||
w13_input_scale, w2_input_scale
|
||||
w13_input_scale,
|
||||
w2_input_scale,
|
||||
layer.moe_config.moe_parallel_config.enable_eplb,
|
||||
)
|
||||
replace_parameter(layer, "w13_input_scale", w13_input_scale)
|
||||
replace_parameter(layer, "w2_input_scale", w2_input_scale)
|
||||
|
||||
@@ -899,7 +899,9 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
|
||||
|
||||
# Per tensor kernels require single activation scale. Use the max.
|
||||
w13_input_scale, w2_input_scale = process_fp8_input_tensor_strategy_moe(
|
||||
w13_input_scale, w2_input_scale
|
||||
w13_input_scale,
|
||||
w2_input_scale,
|
||||
layer.moe_config.moe_parallel_config.enable_eplb,
|
||||
)
|
||||
replace_parameter(layer, "w13_input_scale", w13_input_scale)
|
||||
replace_parameter(layer, "w2_input_scale", w2_input_scale)
|
||||
|
||||
@@ -15,6 +15,9 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
|
||||
swizzle_blockscale,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
amax_for_moe_activation_quant,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
@@ -129,12 +132,13 @@ def prepare_nvfp4_moe_layer_for_flashinfer_cutedsl(
|
||||
and interleaves w13 gate/linear rows for gated activations. Non-gated
|
||||
activations use a single w13 projection and keep its row order unchanged.
|
||||
"""
|
||||
from flashinfer.cute_dsl.utils import convert_sf_to_mma_layout
|
||||
|
||||
# Global scaling factors (same as other FlashInfer backends).
|
||||
num_experts = w13.shape[0]
|
||||
a13_scale = a13_scale.max().to(torch.float32).repeat(num_experts)
|
||||
a2_scale = a2_scale.max().to(torch.float32).repeat(num_experts)
|
||||
enable_eplb = layer.moe_config.moe_parallel_config.enable_eplb
|
||||
a13_scale = amax_for_moe_activation_quant(a13_scale, enable_eplb).repeat(
|
||||
num_experts
|
||||
)
|
||||
a2_scale = amax_for_moe_activation_quant(a2_scale, enable_eplb).repeat(num_experts)
|
||||
|
||||
if layer.activation.is_gated:
|
||||
w13, w13_scale = reorder_w13_to_w31_for_flashinfer_cutedsl(
|
||||
@@ -145,29 +149,8 @@ def prepare_nvfp4_moe_layer_for_flashinfer_cutedsl(
|
||||
w13 = interleave_linear_and_gate(w13, group_size=64, dim=1)
|
||||
w13_scale = interleave_linear_and_gate(w13_scale, group_size=64, dim=1)
|
||||
|
||||
# Convert w13 scale factors: linear → swizzled → MMA layout.
|
||||
w13_scale = swizzle_blockscale(w13_scale)
|
||||
E, M_padded, K_sf_padded = w13_scale.shape
|
||||
w13_scale_flat = w13_scale.reshape(E * M_padded, K_sf_padded)
|
||||
w13_scale = convert_sf_to_mma_layout(
|
||||
w13_scale_flat,
|
||||
m=M_padded,
|
||||
k=K_sf_padded * 16,
|
||||
num_groups=E,
|
||||
sf_vec_size=16,
|
||||
)
|
||||
|
||||
# Convert w2 scale factors: linear → swizzled → MMA layout.
|
||||
w2_scale = swizzle_blockscale(w2_scale)
|
||||
E, M_padded, K_sf_padded = w2_scale.shape
|
||||
w2_scale_flat = w2_scale.reshape(E * M_padded, K_sf_padded)
|
||||
w2_scale = convert_sf_to_mma_layout(
|
||||
w2_scale_flat,
|
||||
m=M_padded,
|
||||
k=K_sf_padded * 16,
|
||||
num_groups=E,
|
||||
sf_vec_size=16,
|
||||
)
|
||||
|
||||
return (
|
||||
w13,
|
||||
@@ -181,6 +164,31 @@ def prepare_nvfp4_moe_layer_for_flashinfer_cutedsl(
|
||||
)
|
||||
|
||||
|
||||
def nvfp4_swizzled_scale_to_cutedsl_mma_view(scale: torch.Tensor) -> torch.Tensor:
|
||||
"""View a swizzled (E, M_padded, K_sf_padded) block-scale tensor in the
|
||||
MMA layout expected by the CuteDSL MoE kernel.
|
||||
|
||||
The returned tensor aliases `scale`'s storage, so in-place updates of the
|
||||
registered Parameter (weight reloads, EPLB rearrangement) are visible to
|
||||
the kernel with no extra bookkeeping.
|
||||
"""
|
||||
from flashinfer.cute_dsl.utils import convert_sf_to_mma_layout
|
||||
|
||||
num_experts, m_padded, k_sf_padded = scale.shape
|
||||
mma_view = convert_sf_to_mma_layout(
|
||||
scale.reshape(num_experts * m_padded, k_sf_padded),
|
||||
m=m_padded,
|
||||
k=k_sf_padded * 16,
|
||||
num_groups=num_experts,
|
||||
sf_vec_size=16,
|
||||
)
|
||||
assert mma_view.data_ptr() == scale.data_ptr(), (
|
||||
"convert_sf_to_mma_layout no longer returns a view of its input; "
|
||||
"the quant config would go stale after weight updates."
|
||||
)
|
||||
return mma_view
|
||||
|
||||
|
||||
def prepare_static_weights_for_trtllm_fp4_moe(
|
||||
# args_dequant,
|
||||
# args,
|
||||
@@ -363,8 +371,13 @@ def prepare_nvfp4_moe_layer_for_fi_or_cutlass(
|
||||
# For some FI kernels, the input scales are shared by all experts.
|
||||
if is_global_sf_supported_for_nvfp4_backend(backend):
|
||||
num_experts = w13.shape[0]
|
||||
a13_scale = a13_scale.max().to(torch.float32).repeat(num_experts)
|
||||
a2_scale = a2_scale.max().to(torch.float32).repeat(num_experts)
|
||||
enable_eplb = layer.moe_config.moe_parallel_config.enable_eplb
|
||||
a13_scale = amax_for_moe_activation_quant(a13_scale, enable_eplb).repeat(
|
||||
num_experts
|
||||
)
|
||||
a2_scale = amax_for_moe_activation_quant(a2_scale, enable_eplb).repeat(
|
||||
num_experts
|
||||
)
|
||||
else:
|
||||
a13_scale = a13_scale.max(dim=1).values.to(torch.float32)
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ import vllm.envs as envs
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
amax_for_moe_activation_quant,
|
||||
get_fp8_min_max,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
|
||||
@@ -1437,6 +1438,7 @@ def process_fp8_weight_tensor_strategy_moe(
|
||||
def process_fp8_input_tensor_strategy_moe(
|
||||
w13_input_scale: torch.Tensor,
|
||||
w2_input_scale: torch.Tensor,
|
||||
enable_eplb: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Process moe input scales for tensor-wise quantization strategy."""
|
||||
|
||||
@@ -1447,4 +1449,7 @@ def process_fp8_input_tensor_strategy_moe(
|
||||
"for each layer."
|
||||
)
|
||||
|
||||
return w13_input_scale.max(), w2_input_scale.max()
|
||||
return (
|
||||
amax_for_moe_activation_quant(w13_input_scale, enable_eplb),
|
||||
amax_for_moe_activation_quant(w2_input_scale, enable_eplb),
|
||||
)
|
||||
|
||||
@@ -62,6 +62,28 @@ def amax_for_moe_weight_quant(amax: torch.Tensor, moe_tp_size: int) -> torch.Ten
|
||||
return amax
|
||||
|
||||
|
||||
def amax_for_moe_activation_quant(
|
||||
a_scale: torch.Tensor, enable_eplb: bool
|
||||
) -> torch.Tensor:
|
||||
"""Reduce a per-expert activation scale to one value shared by all experts.
|
||||
|
||||
Note: when EPLB is enabled and since this quantization scales get
|
||||
folded into the per-expert dequantization alphas, we can only
|
||||
ensure that the quant/dequant scales match by having a single
|
||||
quantization scale shared across all ranks.
|
||||
"""
|
||||
a_max = a_scale.max().to(torch.float32)
|
||||
if enable_eplb:
|
||||
from vllm.distributed.parallel_state import get_ep_group
|
||||
|
||||
torch.distributed.all_reduce(
|
||||
a_max,
|
||||
op=torch.distributed.ReduceOp.MAX,
|
||||
group=get_ep_group().device_group,
|
||||
)
|
||||
return a_max
|
||||
|
||||
|
||||
def get_fp8_min_max() -> tuple[float, float]:
|
||||
"""Get the min and max values for FP8 quantization."""
|
||||
# Using the default value (240.0) from pytorch will cause accuracy
|
||||
|
||||
Reference in New Issue
Block a user