feat: extended EPLB support for Mistral Large 3 and additional MoE backends (#48355)
Signed-off-by: jdebache <jdebache@nvidia.com>
This commit is contained in:
@@ -628,6 +628,17 @@ def make_fp8_moe_quant_config(
|
||||
and block_shape is None
|
||||
):
|
||||
assert a1_scale is not None and a2_scale is not None
|
||||
g1_alphas = w1_scale * a1_scale
|
||||
g2_alphas = w2_scale * a2_scale
|
||||
if layer is not None:
|
||||
layer.register_parameter(
|
||||
"g1_alphas", torch.nn.Parameter(g1_alphas, requires_grad=False)
|
||||
)
|
||||
layer.register_parameter(
|
||||
"g2_alphas", torch.nn.Parameter(g2_alphas, requires_grad=False)
|
||||
)
|
||||
g1_alphas = layer.g1_alphas
|
||||
g2_alphas = layer.g2_alphas
|
||||
return fp8_w8a8_moe_quant_config(
|
||||
w1_scale=w1_scale,
|
||||
w2_scale=w2_scale,
|
||||
@@ -637,8 +648,8 @@ def make_fp8_moe_quant_config(
|
||||
a2_scale=a2_scale,
|
||||
a1_gscale=(1.0 / a1_scale),
|
||||
a2_gscale=(1.0 / a2_scale),
|
||||
g1_alphas=(w1_scale * a1_scale).squeeze(),
|
||||
g2_alphas=(w2_scale * a2_scale).squeeze(),
|
||||
g1_alphas=g1_alphas,
|
||||
g2_alphas=g2_alphas,
|
||||
gemm1_clamp_limit=swiglu_limit,
|
||||
)
|
||||
# MXFP8 (block [1, 32]) dispatches to the mxfp8 activation quant. Scales are
|
||||
|
||||
@@ -19,6 +19,7 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
|
||||
nvfp4_swizzled_scale_to_cutedsl_mma_view,
|
||||
prepare_nvfp4_moe_layer_for_fi_or_cutlass,
|
||||
prepare_nvfp4_moe_layer_for_flashinfer_cutedsl,
|
||||
)
|
||||
@@ -506,6 +507,10 @@ def make_nvfp4_moe_quant_config(
|
||||
gemm1_clamp_limit=swiglu_limit,
|
||||
)
|
||||
|
||||
if backend == NvFp4MoeBackend.FLASHINFER_CUTEDSL:
|
||||
w13_scale = nvfp4_swizzled_scale_to_cutedsl_mma_view(w13_scale)
|
||||
w2_scale = nvfp4_swizzled_scale_to_cutedsl_mma_view(w2_scale)
|
||||
|
||||
# Pass w13_scale_2 / w2_scale_2 directly as g1/g2_alphas.
|
||||
# The expert's process_weights_after_loading will fuse activation
|
||||
# scales in-place. Since the quant config references the same tensor
|
||||
|
||||
+16
-2
@@ -15,6 +15,7 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
|
||||
NvFp4MoeBackend,
|
||||
convert_to_nvfp4_moe_kernel_format,
|
||||
is_global_sf_supported_for_nvfp4_backend,
|
||||
make_nvfp4_moe_kernel,
|
||||
@@ -30,6 +31,15 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
|
||||
|
||||
# NVFP4 backends that have been verified to support EPLB for this quantization recipe.
|
||||
_EPLB_SUPPORTED_NVFP4_BACKENDS = frozenset(
|
||||
{
|
||||
NvFp4MoeBackend.FLASHINFER_CUTEDSL,
|
||||
NvFp4MoeBackend.FLASHINFER_CUTEDSL_BATCHED,
|
||||
NvFp4MoeBackend.FLASHINFER_TRTLLM,
|
||||
}
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -54,6 +64,10 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
self.nvfp4_backend
|
||||
)
|
||||
|
||||
@property
|
||||
def supports_eplb(self) -> bool:
|
||||
return self.nvfp4_backend in _EPLB_SUPPORTED_NVFP4_BACKENDS
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
@@ -223,8 +237,8 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
replace_parameter(layer, "w13_weight_scale", w13_scale)
|
||||
replace_parameter(layer, "w2_weight", w2)
|
||||
replace_parameter(layer, "w2_weight_scale", w2_scale)
|
||||
layer.w13_weight_scale_2 = w13_scale_2
|
||||
layer.w2_weight_scale_2 = w2_scale_2
|
||||
replace_parameter(layer, "w13_weight_scale_2", w13_scale_2)
|
||||
replace_parameter(layer, "w2_weight_scale_2", w2_scale_2)
|
||||
layer.w13_input_scale = a13_scale
|
||||
layer.w2_input_scale = a2_scale
|
||||
|
||||
|
||||
+3
-1
@@ -290,7 +290,9 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
assert self.input_quant.strategy == QuantizationStrategy.TENSOR
|
||||
assert w13_input_scale is not None and w2_input_scale is not None
|
||||
w13_input_scale, w2_input_scale = process_fp8_input_tensor_strategy_moe(
|
||||
w13_input_scale, w2_input_scale
|
||||
w13_input_scale,
|
||||
w2_input_scale,
|
||||
layer.moe_config.moe_parallel_config.enable_eplb,
|
||||
)
|
||||
replace_parameter(layer, "w13_input_scale", w13_input_scale)
|
||||
replace_parameter(layer, "w2_input_scale", w2_input_scale)
|
||||
|
||||
@@ -709,7 +709,9 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
assert not self.block_quant
|
||||
assert w13_input_scale is not None and w2_input_scale is not None
|
||||
w13_input_scale, w2_input_scale = process_fp8_input_tensor_strategy_moe(
|
||||
w13_input_scale, w2_input_scale
|
||||
w13_input_scale,
|
||||
w2_input_scale,
|
||||
layer.moe_config.moe_parallel_config.enable_eplb,
|
||||
)
|
||||
replace_parameter(layer, "w13_input_scale", w13_input_scale)
|
||||
replace_parameter(layer, "w2_input_scale", w2_input_scale)
|
||||
|
||||
@@ -899,7 +899,9 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
|
||||
|
||||
# Per tensor kernels require single activation scale. Use the max.
|
||||
w13_input_scale, w2_input_scale = process_fp8_input_tensor_strategy_moe(
|
||||
w13_input_scale, w2_input_scale
|
||||
w13_input_scale,
|
||||
w2_input_scale,
|
||||
layer.moe_config.moe_parallel_config.enable_eplb,
|
||||
)
|
||||
replace_parameter(layer, "w13_input_scale", w13_input_scale)
|
||||
replace_parameter(layer, "w2_input_scale", w2_input_scale)
|
||||
|
||||
@@ -15,6 +15,9 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
|
||||
swizzle_blockscale,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
amax_for_moe_activation_quant,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
@@ -129,12 +132,13 @@ def prepare_nvfp4_moe_layer_for_flashinfer_cutedsl(
|
||||
and interleaves w13 gate/linear rows for gated activations. Non-gated
|
||||
activations use a single w13 projection and keep its row order unchanged.
|
||||
"""
|
||||
from flashinfer.cute_dsl.utils import convert_sf_to_mma_layout
|
||||
|
||||
# Global scaling factors (same as other FlashInfer backends).
|
||||
num_experts = w13.shape[0]
|
||||
a13_scale = a13_scale.max().to(torch.float32).repeat(num_experts)
|
||||
a2_scale = a2_scale.max().to(torch.float32).repeat(num_experts)
|
||||
enable_eplb = layer.moe_config.moe_parallel_config.enable_eplb
|
||||
a13_scale = amax_for_moe_activation_quant(a13_scale, enable_eplb).repeat(
|
||||
num_experts
|
||||
)
|
||||
a2_scale = amax_for_moe_activation_quant(a2_scale, enable_eplb).repeat(num_experts)
|
||||
|
||||
if layer.activation.is_gated:
|
||||
w13, w13_scale = reorder_w13_to_w31_for_flashinfer_cutedsl(
|
||||
@@ -145,29 +149,8 @@ def prepare_nvfp4_moe_layer_for_flashinfer_cutedsl(
|
||||
w13 = interleave_linear_and_gate(w13, group_size=64, dim=1)
|
||||
w13_scale = interleave_linear_and_gate(w13_scale, group_size=64, dim=1)
|
||||
|
||||
# Convert w13 scale factors: linear → swizzled → MMA layout.
|
||||
w13_scale = swizzle_blockscale(w13_scale)
|
||||
E, M_padded, K_sf_padded = w13_scale.shape
|
||||
w13_scale_flat = w13_scale.reshape(E * M_padded, K_sf_padded)
|
||||
w13_scale = convert_sf_to_mma_layout(
|
||||
w13_scale_flat,
|
||||
m=M_padded,
|
||||
k=K_sf_padded * 16,
|
||||
num_groups=E,
|
||||
sf_vec_size=16,
|
||||
)
|
||||
|
||||
# Convert w2 scale factors: linear → swizzled → MMA layout.
|
||||
w2_scale = swizzle_blockscale(w2_scale)
|
||||
E, M_padded, K_sf_padded = w2_scale.shape
|
||||
w2_scale_flat = w2_scale.reshape(E * M_padded, K_sf_padded)
|
||||
w2_scale = convert_sf_to_mma_layout(
|
||||
w2_scale_flat,
|
||||
m=M_padded,
|
||||
k=K_sf_padded * 16,
|
||||
num_groups=E,
|
||||
sf_vec_size=16,
|
||||
)
|
||||
|
||||
return (
|
||||
w13,
|
||||
@@ -181,6 +164,31 @@ def prepare_nvfp4_moe_layer_for_flashinfer_cutedsl(
|
||||
)
|
||||
|
||||
|
||||
def nvfp4_swizzled_scale_to_cutedsl_mma_view(scale: torch.Tensor) -> torch.Tensor:
|
||||
"""View a swizzled (E, M_padded, K_sf_padded) block-scale tensor in the
|
||||
MMA layout expected by the CuteDSL MoE kernel.
|
||||
|
||||
The returned tensor aliases `scale`'s storage, so in-place updates of the
|
||||
registered Parameter (weight reloads, EPLB rearrangement) are visible to
|
||||
the kernel with no extra bookkeeping.
|
||||
"""
|
||||
from flashinfer.cute_dsl.utils import convert_sf_to_mma_layout
|
||||
|
||||
num_experts, m_padded, k_sf_padded = scale.shape
|
||||
mma_view = convert_sf_to_mma_layout(
|
||||
scale.reshape(num_experts * m_padded, k_sf_padded),
|
||||
m=m_padded,
|
||||
k=k_sf_padded * 16,
|
||||
num_groups=num_experts,
|
||||
sf_vec_size=16,
|
||||
)
|
||||
assert mma_view.data_ptr() == scale.data_ptr(), (
|
||||
"convert_sf_to_mma_layout no longer returns a view of its input; "
|
||||
"the quant config would go stale after weight updates."
|
||||
)
|
||||
return mma_view
|
||||
|
||||
|
||||
def prepare_static_weights_for_trtllm_fp4_moe(
|
||||
# args_dequant,
|
||||
# args,
|
||||
@@ -363,8 +371,13 @@ def prepare_nvfp4_moe_layer_for_fi_or_cutlass(
|
||||
# For some FI kernels, the input scales are shared by all experts.
|
||||
if is_global_sf_supported_for_nvfp4_backend(backend):
|
||||
num_experts = w13.shape[0]
|
||||
a13_scale = a13_scale.max().to(torch.float32).repeat(num_experts)
|
||||
a2_scale = a2_scale.max().to(torch.float32).repeat(num_experts)
|
||||
enable_eplb = layer.moe_config.moe_parallel_config.enable_eplb
|
||||
a13_scale = amax_for_moe_activation_quant(a13_scale, enable_eplb).repeat(
|
||||
num_experts
|
||||
)
|
||||
a2_scale = amax_for_moe_activation_quant(a2_scale, enable_eplb).repeat(
|
||||
num_experts
|
||||
)
|
||||
else:
|
||||
a13_scale = a13_scale.max(dim=1).values.to(torch.float32)
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ import vllm.envs as envs
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
amax_for_moe_activation_quant,
|
||||
get_fp8_min_max,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
|
||||
@@ -1437,6 +1438,7 @@ def process_fp8_weight_tensor_strategy_moe(
|
||||
def process_fp8_input_tensor_strategy_moe(
|
||||
w13_input_scale: torch.Tensor,
|
||||
w2_input_scale: torch.Tensor,
|
||||
enable_eplb: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Process moe input scales for tensor-wise quantization strategy."""
|
||||
|
||||
@@ -1447,4 +1449,7 @@ def process_fp8_input_tensor_strategy_moe(
|
||||
"for each layer."
|
||||
)
|
||||
|
||||
return w13_input_scale.max(), w2_input_scale.max()
|
||||
return (
|
||||
amax_for_moe_activation_quant(w13_input_scale, enable_eplb),
|
||||
amax_for_moe_activation_quant(w2_input_scale, enable_eplb),
|
||||
)
|
||||
|
||||
@@ -62,6 +62,28 @@ def amax_for_moe_weight_quant(amax: torch.Tensor, moe_tp_size: int) -> torch.Ten
|
||||
return amax
|
||||
|
||||
|
||||
def amax_for_moe_activation_quant(
|
||||
a_scale: torch.Tensor, enable_eplb: bool
|
||||
) -> torch.Tensor:
|
||||
"""Reduce a per-expert activation scale to one value shared by all experts.
|
||||
|
||||
Note: when EPLB is enabled and since this quantization scales get
|
||||
folded into the per-expert dequantization alphas, we can only
|
||||
ensure that the quant/dequant scales match by having a single
|
||||
quantization scale shared across all ranks.
|
||||
"""
|
||||
a_max = a_scale.max().to(torch.float32)
|
||||
if enable_eplb:
|
||||
from vllm.distributed.parallel_state import get_ep_group
|
||||
|
||||
torch.distributed.all_reduce(
|
||||
a_max,
|
||||
op=torch.distributed.ReduceOp.MAX,
|
||||
group=get_ep_group().device_group,
|
||||
)
|
||||
return a_max
|
||||
|
||||
|
||||
def get_fp8_min_max() -> tuple[float, float]:
|
||||
"""Get the min and max values for FP8 quantization."""
|
||||
# Using the default value (240.0) from pytorch will cause accuracy
|
||||
|
||||
@@ -5,7 +5,6 @@ import asyncio
|
||||
from collections.abc import (
|
||||
AsyncGenerator,
|
||||
Callable,
|
||||
Iterable,
|
||||
Mapping,
|
||||
MutableSequence,
|
||||
Sequence,
|
||||
@@ -949,7 +948,7 @@ class MixtureOfExperts(Protocol):
|
||||
num_redundant_experts: int
|
||||
"""Number of redundant experts in this model."""
|
||||
|
||||
moe_layers: Iterable["MoERunner"]
|
||||
moe_layers: Sequence["MoERunner"]
|
||||
"""List of MoE layers in this model."""
|
||||
|
||||
def set_eplb_state(
|
||||
@@ -992,6 +991,37 @@ class MixtureOfExperts(Protocol):
|
||||
) -> None: ...
|
||||
|
||||
|
||||
def get_mixture_of_experts_model(model: object) -> MixtureOfExperts | None:
|
||||
"""Return the MixtureOfExperts contained within an arbitrary model.
|
||||
|
||||
- If the model itself is a MixtureOfExperts, return the model directly.
|
||||
- If the model is a multi-modal model, and its `language_model` is a
|
||||
MixtureOfExperts, return the `language_model`.
|
||||
- If neither, return None.
|
||||
|
||||
Args:
|
||||
model: Model being served.
|
||||
|
||||
Returns:
|
||||
The MixtureOfExperts instance contained within the model, or None.
|
||||
"""
|
||||
|
||||
if is_mixture_of_experts(model):
|
||||
return model
|
||||
|
||||
if isinstance(model, SupportsMultiModal):
|
||||
try:
|
||||
mm_language_model = model.get_language_model()
|
||||
return (
|
||||
mm_language_model if is_mixture_of_experts(mm_language_model) else None
|
||||
)
|
||||
except NotImplementedError:
|
||||
logger.info_once("Cannot fetch language_model from MultiModal model")
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def is_mixture_of_experts(model: object) -> TypeIs[MixtureOfExperts]:
|
||||
return (
|
||||
isinstance(model, MixtureOfExperts) and getattr(model, "num_moe_layers", 0) > 0
|
||||
|
||||
Reference in New Issue
Block a user