[Bugfix] Allow skipping MoE in NVFP4 (fix for MTP) (#25987)

Signed-off-by: Benjamin Chislett <bchislett@nvidia.com>
This commit is contained in:
Benjamin Chislett
2025-10-06 16:16:30 -04:00
committed by GitHub
parent f23b4c04fd
commit 2161efe978
5 changed files with 18 additions and 5 deletions
@@ -884,8 +884,9 @@ class ModelOptNvFp4Config(QuantizationConfig):
) -> Optional["QuantizeMethodBase"]:
from vllm.attention.layer import Attention # Avoid circular import
skip_layer = self.is_layer_excluded(prefix)
if isinstance(layer, LinearBase):
if self.is_layer_excluded(prefix):
if skip_layer:
return UnquantizedLinearMethod()
# Check if this is a vision model layer that should not be quantized
if "vision_tower" in prefix or "vision_model" in prefix:
@@ -894,6 +895,8 @@ class ModelOptNvFp4Config(QuantizationConfig):
elif isinstance(layer, Attention):
return ModelOptFp8KVCacheMethod(self)
elif isinstance(layer, FusedMoE):
if skip_layer:
return None
return ModelOptNvFp4FusedMoE(self, layer.moe_config, layer)
return None