[Models] Fix MiMo v2.x QKV TP sharding + FP4 support (#45200)
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
64833f8158
commit
b5adb027ad
@@ -106,6 +106,7 @@ class Fp8Config(QuantizationConfig):
|
||||
activation_scheme: str = "dynamic",
|
||||
ignored_layers: list[str] | None = None,
|
||||
weight_block_size: list[int] | None = None,
|
||||
store_dtype: str | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
@@ -115,6 +116,7 @@ class Fp8Config(QuantizationConfig):
|
||||
raise ValueError(f"Unsupported activation scheme {activation_scheme}")
|
||||
self.activation_scheme = activation_scheme
|
||||
self.ignored_layers = ignored_layers or []
|
||||
self.store_dtype = store_dtype
|
||||
if weight_block_size is not None:
|
||||
if not is_checkpoint_fp8_serialized:
|
||||
raise ValueError(
|
||||
@@ -162,6 +164,7 @@ class Fp8Config(QuantizationConfig):
|
||||
activation_scheme = cls.get_from_keys(config, ["activation_scheme"])
|
||||
ignored_layers = cls.get_from_keys_or(config, ["ignored_layers"], None)
|
||||
weight_block_size = cls.get_from_keys_or(config, ["weight_block_size"], None)
|
||||
store_dtype = cls.get_from_keys_or(config, ["store_dtype"], None)
|
||||
if not ignored_layers:
|
||||
ignored_layers = cls.get_from_keys_or(
|
||||
config, ["modules_to_not_convert"], None
|
||||
@@ -171,6 +174,7 @@ class Fp8Config(QuantizationConfig):
|
||||
activation_scheme=activation_scheme,
|
||||
ignored_layers=ignored_layers,
|
||||
weight_block_size=weight_block_size,
|
||||
store_dtype=store_dtype,
|
||||
)
|
||||
|
||||
def get_quant_method(
|
||||
@@ -198,6 +202,12 @@ class Fp8Config(QuantizationConfig):
|
||||
fused_mapping=self.packed_modules_mapping,
|
||||
):
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
if self.store_dtype == "mxfp4":
|
||||
from vllm.model_executor.layers.quantization.mxfp4 import (
|
||||
Mxfp4MoEMethod,
|
||||
)
|
||||
|
||||
return Mxfp4MoEMethod(layer.moe_config)
|
||||
if self.is_checkpoint_fp8_serialized:
|
||||
moe_quant_method = Fp8MoEMethod(self, layer)
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user