[Models] Fix MiMo v2.x QKV TP sharding + FP4 support (#45200)

Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Giancarlo Delfin
2026-06-15 15:13:34 +08:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 64833f8158
commit b5adb027ad
2 changed files with 170 additions and 5 deletions
@@ -106,6 +106,7 @@ class Fp8Config(QuantizationConfig):
activation_scheme: str = "dynamic",
ignored_layers: list[str] | None = None,
weight_block_size: list[int] | None = None,
store_dtype: str | None = None,
) -> None:
super().__init__()
@@ -115,6 +116,7 @@ class Fp8Config(QuantizationConfig):
raise ValueError(f"Unsupported activation scheme {activation_scheme}")
self.activation_scheme = activation_scheme
self.ignored_layers = ignored_layers or []
self.store_dtype = store_dtype
if weight_block_size is not None:
if not is_checkpoint_fp8_serialized:
raise ValueError(
@@ -162,6 +164,7 @@ class Fp8Config(QuantizationConfig):
activation_scheme = cls.get_from_keys(config, ["activation_scheme"])
ignored_layers = cls.get_from_keys_or(config, ["ignored_layers"], None)
weight_block_size = cls.get_from_keys_or(config, ["weight_block_size"], None)
store_dtype = cls.get_from_keys_or(config, ["store_dtype"], None)
if not ignored_layers:
ignored_layers = cls.get_from_keys_or(
config, ["modules_to_not_convert"], None
@@ -171,6 +174,7 @@ class Fp8Config(QuantizationConfig):
activation_scheme=activation_scheme,
ignored_layers=ignored_layers,
weight_block_size=weight_block_size,
store_dtype=store_dtype,
)
def get_quant_method(
@@ -198,6 +202,12 @@ class Fp8Config(QuantizationConfig):
fused_mapping=self.packed_modules_mapping,
):
return UnquantizedFusedMoEMethod(layer.moe_config)
if self.store_dtype == "mxfp4":
from vllm.model_executor.layers.quantization.mxfp4 import (
Mxfp4MoEMethod,
)
return Mxfp4MoEMethod(layer.moe_config)
if self.is_checkpoint_fp8_serialized:
moe_quant_method = Fp8MoEMethod(self, layer)
else: