[Bugfix] Allow skipping MoE in NVFP4 (fix for MTP) (#25987)
Signed-off-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
@@ -1194,6 +1194,8 @@ class FusedMoE(CustomOp):
|
||||
if quant_config is None
|
||||
else quant_config.get_quant_method(self, prefix)
|
||||
)
|
||||
if quant_method is None:
|
||||
quant_method = UnquantizedFusedMoEMethod(moe)
|
||||
|
||||
assert quant_method is not None
|
||||
assert isinstance(quant_method, FusedMoEMethodBase)
|
||||
|
||||
@@ -884,8 +884,9 @@ class ModelOptNvFp4Config(QuantizationConfig):
|
||||
) -> Optional["QuantizeMethodBase"]:
|
||||
from vllm.attention.layer import Attention # Avoid circular import
|
||||
|
||||
skip_layer = self.is_layer_excluded(prefix)
|
||||
if isinstance(layer, LinearBase):
|
||||
if self.is_layer_excluded(prefix):
|
||||
if skip_layer:
|
||||
return UnquantizedLinearMethod()
|
||||
# Check if this is a vision model layer that should not be quantized
|
||||
if "vision_tower" in prefix or "vision_model" in prefix:
|
||||
@@ -894,6 +895,8 @@ class ModelOptNvFp4Config(QuantizationConfig):
|
||||
elif isinstance(layer, Attention):
|
||||
return ModelOptFp8KVCacheMethod(self)
|
||||
elif isinstance(layer, FusedMoE):
|
||||
if skip_layer:
|
||||
return None
|
||||
return ModelOptNvFp4FusedMoE(self, layer.moe_config, layer)
|
||||
return None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user