diff --git a/tests/quantization/test_modelopt.py b/tests/quantization/test_modelopt.py index 593075e9d4..a11d5d9e66 100644 --- a/tests/quantization/test_modelopt.py +++ b/tests/quantization/test_modelopt.py @@ -6,7 +6,8 @@ Run `pytest tests/quantization/test_modelopt.py`. """ import os -from typing import NoReturn +from typing import Any, NoReturn +from unittest.mock import MagicMock, patch import pytest import torch @@ -285,3 +286,141 @@ def test_modelopt_nvfp4_config_dispatches_w4a16_method(): assert config.LinearMethodCls is ModelOptNvFp4W4A16LinearMethod assert config.LinearMethodCls is not ModelOptNvFp4LinearMethod assert config.quant_method == "W4A16_NVFP4" + + +@pytest.mark.parametrize( + "quant_method, expected_use_a16, act_key_is_none", + [ + ("NVFP4", False, False), # W4A4 default + ("W4A16_NVFP4", True, True), # native W4A16 ckpt + ], +) +def test_modelopt_nvfp4_moe_dispatches_to_marlin_when_w4a16( + quant_method, expected_use_a16, act_key_is_none +): + """``ModelOptNvFp4FusedMoE``: when the ckpt's ``quant_method`` is + ``W4A16_NVFP4``, the MoE class must pass ``activation_key=None`` to + ``select_nvfp4_moe_backend``. That filters out every W4A4 backend + (their ``_supports_quant_scheme`` requires + ``(kNvfp4Static, kNvfp4Dynamic)`` exactly); Marlin survives because + it only checks ``weight_key``. A regression here would mean a W4A16 + ckpt silently went to the cutlass W4A4 path. + """ + from vllm.model_executor.layers.quantization.modelopt import ( + ModelOptNvFp4Config, + ModelOptNvFp4FusedMoE, + ) + from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kNvfp4Dynamic, + kNvfp4Static, + ) + + config = ModelOptNvFp4Config( + quant_method=quant_method, + is_checkpoint_nvfp4_serialized=True, + kv_cache_quant_algo=None, + exclude_modules=[], + group_size=16, + ) + + mock_select = MagicMock(return_value=(MagicMock(), MagicMock())) + with ( + patch( + "vllm.model_executor.layers.quantization.modelopt.select_nvfp4_moe_backend", + mock_select, + ), + patch( + "vllm.model_executor.layers.quantization.modelopt." + "is_global_sf_supported_for_nvfp4_backend", + return_value=False, + ), + ): + moe = ModelOptNvFp4FusedMoE(config, MagicMock()) + + assert moe.use_a16 is expected_use_a16 + _, kwargs = mock_select.call_args + assert kwargs["weight_key"] is kNvfp4Static + if act_key_is_none: + assert kwargs["activation_key"] is None + else: + assert kwargs["activation_key"] is kNvfp4Dynamic + + +@pytest.mark.parametrize( + "per_layer_algo, expected_linear_cls_name", + [ + ("NVFP4", "ModelOptNvFp4LinearMethod"), + ("W4A16_NVFP4", "ModelOptNvFp4W4A16LinearMethod"), + ], +) +def test_modelopt_mixed_precision_dispatches_w4a16_layer( + per_layer_algo, expected_linear_cls_name +): + """``ModelOptMixedPrecisionConfig.get_quant_method`` must route a Linear + layer to the right LinearMethod based on its per-layer ``quant_algo`` + entry in ``quantized_layers``. Verifies the new ``W4A16_NVFP4`` branch + coexists with the existing ``NVFP4`` branch without regression. A + regression here would mean a W4A16 layer in a mixed-precision ckpt + silently fell through to ``UnquantizedLinearMethod``. + + NOTE: FP8 dispatch (the third branch of get_quant_method) is not + covered here because ``ModelOptFp8LinearMethod.__init__`` reads + ``get_current_vllm_config().model_config.dtype``, which requires a + fully constructed ``ModelConfig`` (real model path). FP8 routing in + mixed-precision is exercised by the existing integration tests + above that use the ``vllm_runner`` fixture (e.g. + ``test_modelopt_fp8_checkpoint_setup``). Our PR doesn't change the + FP8 branch, so this isn't a coverage gap. + """ + from vllm.model_executor.layers.linear import LinearBase + from vllm.model_executor.layers.quantization import modelopt as m + + hf_quant_config: dict[str, Any] = { + "quantization": { + "quant_algo": "MIXED_PRECISION", + "kv_cache_quant_algo": None, + "exclude_modules": [], + "group_size": 16, + "quantized_layers": { + "model.layers.0.fake_proj": {"quant_algo": per_layer_algo}, + }, + } + } + config = m.ModelOptMixedPrecisionConfig.from_config(hf_quant_config) + + fake_layer = MagicMock(spec=LinearBase) + method = config.get_quant_method(fake_layer, "model.layers.0.fake_proj") + + expected_cls = getattr(m, expected_linear_cls_name) + assert isinstance(method, expected_cls), ( + f"Expected {expected_linear_cls_name}, got {type(method).__name__}" + ) + + +def test_modelopt_mixed_precision_builds_w4a16_sibling_config(): + """Sanity: ``ModelOptMixedPrecisionConfig._from_config`` builds **two** + NVFP4 sub-configs — one for W4A4 (default) and one tagged + ``quant_method='W4A16_NVFP4'`` — so per-layer dispatch can hand + Marlin-bound layers the right config without re-instantiating it on + every call. + """ + from vllm.model_executor.layers.quantization import modelopt as m + + hf_quant_config: dict[str, Any] = { + "quantization": { + "quant_algo": "MIXED_PRECISION", + "kv_cache_quant_algo": None, + "exclude_modules": [], + "group_size": 16, + "quantized_layers": { + "model.layers.0.a": {"quant_algo": "NVFP4"}, + "model.layers.0.b": {"quant_algo": "W4A16_NVFP4"}, + }, + } + } + config = m.ModelOptMixedPrecisionConfig.from_config(hf_quant_config) + + assert config.nvfp4_config.quant_method == "NVFP4" + assert config.nvfp4_config.LinearMethodCls is m.ModelOptNvFp4LinearMethod + assert config.w4a16_nvfp4_config.quant_method == "W4A16_NVFP4" + assert config.w4a16_nvfp4_config.LinearMethodCls is m.ModelOptNvFp4W4A16LinearMethod diff --git a/vllm/model_executor/layers/quantization/modelopt.py b/vllm/model_executor/layers/quantization/modelopt.py index 67cdab4f22..5353096ff7 100644 --- a/vllm/model_executor/layers/quantization/modelopt.py +++ b/vllm/model_executor/layers/quantization/modelopt.py @@ -1392,11 +1392,17 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase): ) -> None: super().__init__(moe_config) self.quant_config = quant_config - # Select experts implementation. + # W4A16 mode fires for W4A16_NVFP4 on-disk checkpoints. With + # activation_key=None every W4A4 backend's _supports_quant_scheme + # rejects itself (they all require (kNvfp4Static, kNvfp4Dynamic) + # exactly); only Marlin survives. Marlin's MoE path drops + # activation scales in convert_to_nvfp4_moe_kernel_format, so no + # other change is needed. + self.use_a16 = quant_config.quant_method == "W4A16_NVFP4" self.nvfp4_backend, self.experts_cls = select_nvfp4_moe_backend( config=self.moe, weight_key=kNvfp4Static, - activation_key=kNvfp4Dynamic, + activation_key=None if self.use_a16 else kNvfp4Dynamic, ) self.use_global_sf = is_global_sf_supported_for_nvfp4_backend( @@ -2211,12 +2217,14 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase): quantized_layers: dict[str, dict[str, Any]], fp8_config: ModelOptFp8Config, nvfp4_config: ModelOptNvFp4Config, + w4a16_nvfp4_config: ModelOptNvFp4Config, ) -> None: super().__init__(exclude_modules) self.kv_cache_quant_method = kv_cache_quant_method self.quantized_layers = quantized_layers self.fp8_config = fp8_config self.nvfp4_config = nvfp4_config + self.w4a16_nvfp4_config = w4a16_nvfp4_config def get_name(self) -> QuantizationMethods: return "modelopt_mixed" @@ -2261,10 +2269,15 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase): "'quantized_layers' mapping in the quantization config." ) - # Determine group_size from the first NVFP4 entry if not provided. + # Determine group_size from the first NVFP4-family entry if not + # provided. Both NVFP4 (W4A4) and W4A16_NVFP4 share the same packing + # + group-size convention; either entry resolves the value. if group_size is None: for layer_info in quantized_layers.values(): - if layer_info.get("quant_algo", "").upper() == "NVFP4": + if layer_info.get("quant_algo", "").upper() in ( + "NVFP4", + "W4A16_NVFP4", + ): group_size = layer_info.get("group_size", 16) break if group_size is None: @@ -2282,6 +2295,18 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase): exclude_modules=[], group_size=group_size, ) + # Sibling config for layers that declare quant_algo: "W4A16_NVFP4". + # ModelOptNvFp4Config.__init__ keys LinearMethodCls off quant_method, + # so this instance auto-selects ModelOptNvFp4W4A16LinearMethod. The + # MoE side reads quant_config.quant_method == "W4A16_NVFP4" to set + # use_a16 → Marlin backend in ModelOptNvFp4FusedMoE.__init__. + w4a16_nvfp4_config = ModelOptNvFp4Config( + quant_method="W4A16_NVFP4", + is_checkpoint_nvfp4_serialized=True, + kv_cache_quant_algo=kv_cache_quant_method, + exclude_modules=[], + group_size=group_size, + ) return cls( kv_cache_quant_method=kv_cache_quant_method, @@ -2289,6 +2314,7 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase): quantized_layers=quantized_layers, fp8_config=fp8_config, nvfp4_config=nvfp4_config, + w4a16_nvfp4_config=w4a16_nvfp4_config, ) def _resolve_quant_algo(self, prefix: str) -> str | None: @@ -2355,6 +2381,8 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase): return ModelOptFp8LinearMethod(self.fp8_config) if quant_algo == "NVFP4": return ModelOptNvFp4LinearMethod(self.nvfp4_config) + if quant_algo == "W4A16_NVFP4": + return ModelOptNvFp4W4A16LinearMethod(self.w4a16_nvfp4_config) # Layer not in quantized_layers — leave unquantized return UnquantizedLinearMethod() @@ -2369,6 +2397,11 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase): quant_config=self.nvfp4_config, moe_config=layer.moe_config, ) + if quant_algo == "W4A16_NVFP4": + return ModelOptNvFp4FusedMoE( + quant_config=self.w4a16_nvfp4_config, + moe_config=layer.moe_config, + ) return None return None