[Quantization][ModelOpt] W4A16 NVFP4 fused MoE + mixed-precision dispatch (#42566)
Signed-off-by: Juhi Mittal <juhim@nvidia.com>
This commit is contained in:
@@ -6,7 +6,8 @@ Run `pytest tests/quantization/test_modelopt.py`.
|
||||
"""
|
||||
|
||||
import os
|
||||
from typing import NoReturn
|
||||
from typing import Any, NoReturn
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -285,3 +286,141 @@ def test_modelopt_nvfp4_config_dispatches_w4a16_method():
|
||||
assert config.LinearMethodCls is ModelOptNvFp4W4A16LinearMethod
|
||||
assert config.LinearMethodCls is not ModelOptNvFp4LinearMethod
|
||||
assert config.quant_method == "W4A16_NVFP4"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"quant_method, expected_use_a16, act_key_is_none",
|
||||
[
|
||||
("NVFP4", False, False), # W4A4 default
|
||||
("W4A16_NVFP4", True, True), # native W4A16 ckpt
|
||||
],
|
||||
)
|
||||
def test_modelopt_nvfp4_moe_dispatches_to_marlin_when_w4a16(
|
||||
quant_method, expected_use_a16, act_key_is_none
|
||||
):
|
||||
"""``ModelOptNvFp4FusedMoE``: when the ckpt's ``quant_method`` is
|
||||
``W4A16_NVFP4``, the MoE class must pass ``activation_key=None`` to
|
||||
``select_nvfp4_moe_backend``. That filters out every W4A4 backend
|
||||
(their ``_supports_quant_scheme`` requires
|
||||
``(kNvfp4Static, kNvfp4Dynamic)`` exactly); Marlin survives because
|
||||
it only checks ``weight_key``. A regression here would mean a W4A16
|
||||
ckpt silently went to the cutlass W4A4 path.
|
||||
"""
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptNvFp4Config,
|
||||
ModelOptNvFp4FusedMoE,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kNvfp4Dynamic,
|
||||
kNvfp4Static,
|
||||
)
|
||||
|
||||
config = ModelOptNvFp4Config(
|
||||
quant_method=quant_method,
|
||||
is_checkpoint_nvfp4_serialized=True,
|
||||
kv_cache_quant_algo=None,
|
||||
exclude_modules=[],
|
||||
group_size=16,
|
||||
)
|
||||
|
||||
mock_select = MagicMock(return_value=(MagicMock(), MagicMock()))
|
||||
with (
|
||||
patch(
|
||||
"vllm.model_executor.layers.quantization.modelopt.select_nvfp4_moe_backend",
|
||||
mock_select,
|
||||
),
|
||||
patch(
|
||||
"vllm.model_executor.layers.quantization.modelopt."
|
||||
"is_global_sf_supported_for_nvfp4_backend",
|
||||
return_value=False,
|
||||
),
|
||||
):
|
||||
moe = ModelOptNvFp4FusedMoE(config, MagicMock())
|
||||
|
||||
assert moe.use_a16 is expected_use_a16
|
||||
_, kwargs = mock_select.call_args
|
||||
assert kwargs["weight_key"] is kNvfp4Static
|
||||
if act_key_is_none:
|
||||
assert kwargs["activation_key"] is None
|
||||
else:
|
||||
assert kwargs["activation_key"] is kNvfp4Dynamic
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"per_layer_algo, expected_linear_cls_name",
|
||||
[
|
||||
("NVFP4", "ModelOptNvFp4LinearMethod"),
|
||||
("W4A16_NVFP4", "ModelOptNvFp4W4A16LinearMethod"),
|
||||
],
|
||||
)
|
||||
def test_modelopt_mixed_precision_dispatches_w4a16_layer(
|
||||
per_layer_algo, expected_linear_cls_name
|
||||
):
|
||||
"""``ModelOptMixedPrecisionConfig.get_quant_method`` must route a Linear
|
||||
layer to the right LinearMethod based on its per-layer ``quant_algo``
|
||||
entry in ``quantized_layers``. Verifies the new ``W4A16_NVFP4`` branch
|
||||
coexists with the existing ``NVFP4`` branch without regression. A
|
||||
regression here would mean a W4A16 layer in a mixed-precision ckpt
|
||||
silently fell through to ``UnquantizedLinearMethod``.
|
||||
|
||||
NOTE: FP8 dispatch (the third branch of get_quant_method) is not
|
||||
covered here because ``ModelOptFp8LinearMethod.__init__`` reads
|
||||
``get_current_vllm_config().model_config.dtype``, which requires a
|
||||
fully constructed ``ModelConfig`` (real model path). FP8 routing in
|
||||
mixed-precision is exercised by the existing integration tests
|
||||
above that use the ``vllm_runner`` fixture (e.g.
|
||||
``test_modelopt_fp8_checkpoint_setup``). Our PR doesn't change the
|
||||
FP8 branch, so this isn't a coverage gap.
|
||||
"""
|
||||
from vllm.model_executor.layers.linear import LinearBase
|
||||
from vllm.model_executor.layers.quantization import modelopt as m
|
||||
|
||||
hf_quant_config: dict[str, Any] = {
|
||||
"quantization": {
|
||||
"quant_algo": "MIXED_PRECISION",
|
||||
"kv_cache_quant_algo": None,
|
||||
"exclude_modules": [],
|
||||
"group_size": 16,
|
||||
"quantized_layers": {
|
||||
"model.layers.0.fake_proj": {"quant_algo": per_layer_algo},
|
||||
},
|
||||
}
|
||||
}
|
||||
config = m.ModelOptMixedPrecisionConfig.from_config(hf_quant_config)
|
||||
|
||||
fake_layer = MagicMock(spec=LinearBase)
|
||||
method = config.get_quant_method(fake_layer, "model.layers.0.fake_proj")
|
||||
|
||||
expected_cls = getattr(m, expected_linear_cls_name)
|
||||
assert isinstance(method, expected_cls), (
|
||||
f"Expected {expected_linear_cls_name}, got {type(method).__name__}"
|
||||
)
|
||||
|
||||
|
||||
def test_modelopt_mixed_precision_builds_w4a16_sibling_config():
|
||||
"""Sanity: ``ModelOptMixedPrecisionConfig._from_config`` builds **two**
|
||||
NVFP4 sub-configs — one for W4A4 (default) and one tagged
|
||||
``quant_method='W4A16_NVFP4'`` — so per-layer dispatch can hand
|
||||
Marlin-bound layers the right config without re-instantiating it on
|
||||
every call.
|
||||
"""
|
||||
from vllm.model_executor.layers.quantization import modelopt as m
|
||||
|
||||
hf_quant_config: dict[str, Any] = {
|
||||
"quantization": {
|
||||
"quant_algo": "MIXED_PRECISION",
|
||||
"kv_cache_quant_algo": None,
|
||||
"exclude_modules": [],
|
||||
"group_size": 16,
|
||||
"quantized_layers": {
|
||||
"model.layers.0.a": {"quant_algo": "NVFP4"},
|
||||
"model.layers.0.b": {"quant_algo": "W4A16_NVFP4"},
|
||||
},
|
||||
}
|
||||
}
|
||||
config = m.ModelOptMixedPrecisionConfig.from_config(hf_quant_config)
|
||||
|
||||
assert config.nvfp4_config.quant_method == "NVFP4"
|
||||
assert config.nvfp4_config.LinearMethodCls is m.ModelOptNvFp4LinearMethod
|
||||
assert config.w4a16_nvfp4_config.quant_method == "W4A16_NVFP4"
|
||||
assert config.w4a16_nvfp4_config.LinearMethodCls is m.ModelOptNvFp4W4A16LinearMethod
|
||||
|
||||
@@ -1392,11 +1392,17 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
) -> None:
|
||||
super().__init__(moe_config)
|
||||
self.quant_config = quant_config
|
||||
# Select experts implementation.
|
||||
# W4A16 mode fires for W4A16_NVFP4 on-disk checkpoints. With
|
||||
# activation_key=None every W4A4 backend's _supports_quant_scheme
|
||||
# rejects itself (they all require (kNvfp4Static, kNvfp4Dynamic)
|
||||
# exactly); only Marlin survives. Marlin's MoE path drops
|
||||
# activation scales in convert_to_nvfp4_moe_kernel_format, so no
|
||||
# other change is needed.
|
||||
self.use_a16 = quant_config.quant_method == "W4A16_NVFP4"
|
||||
self.nvfp4_backend, self.experts_cls = select_nvfp4_moe_backend(
|
||||
config=self.moe,
|
||||
weight_key=kNvfp4Static,
|
||||
activation_key=kNvfp4Dynamic,
|
||||
activation_key=None if self.use_a16 else kNvfp4Dynamic,
|
||||
)
|
||||
|
||||
self.use_global_sf = is_global_sf_supported_for_nvfp4_backend(
|
||||
@@ -2211,12 +2217,14 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
|
||||
quantized_layers: dict[str, dict[str, Any]],
|
||||
fp8_config: ModelOptFp8Config,
|
||||
nvfp4_config: ModelOptNvFp4Config,
|
||||
w4a16_nvfp4_config: ModelOptNvFp4Config,
|
||||
) -> None:
|
||||
super().__init__(exclude_modules)
|
||||
self.kv_cache_quant_method = kv_cache_quant_method
|
||||
self.quantized_layers = quantized_layers
|
||||
self.fp8_config = fp8_config
|
||||
self.nvfp4_config = nvfp4_config
|
||||
self.w4a16_nvfp4_config = w4a16_nvfp4_config
|
||||
|
||||
def get_name(self) -> QuantizationMethods:
|
||||
return "modelopt_mixed"
|
||||
@@ -2261,10 +2269,15 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
|
||||
"'quantized_layers' mapping in the quantization config."
|
||||
)
|
||||
|
||||
# Determine group_size from the first NVFP4 entry if not provided.
|
||||
# Determine group_size from the first NVFP4-family entry if not
|
||||
# provided. Both NVFP4 (W4A4) and W4A16_NVFP4 share the same packing
|
||||
# + group-size convention; either entry resolves the value.
|
||||
if group_size is None:
|
||||
for layer_info in quantized_layers.values():
|
||||
if layer_info.get("quant_algo", "").upper() == "NVFP4":
|
||||
if layer_info.get("quant_algo", "").upper() in (
|
||||
"NVFP4",
|
||||
"W4A16_NVFP4",
|
||||
):
|
||||
group_size = layer_info.get("group_size", 16)
|
||||
break
|
||||
if group_size is None:
|
||||
@@ -2282,6 +2295,18 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
|
||||
exclude_modules=[],
|
||||
group_size=group_size,
|
||||
)
|
||||
# Sibling config for layers that declare quant_algo: "W4A16_NVFP4".
|
||||
# ModelOptNvFp4Config.__init__ keys LinearMethodCls off quant_method,
|
||||
# so this instance auto-selects ModelOptNvFp4W4A16LinearMethod. The
|
||||
# MoE side reads quant_config.quant_method == "W4A16_NVFP4" to set
|
||||
# use_a16 → Marlin backend in ModelOptNvFp4FusedMoE.__init__.
|
||||
w4a16_nvfp4_config = ModelOptNvFp4Config(
|
||||
quant_method="W4A16_NVFP4",
|
||||
is_checkpoint_nvfp4_serialized=True,
|
||||
kv_cache_quant_algo=kv_cache_quant_method,
|
||||
exclude_modules=[],
|
||||
group_size=group_size,
|
||||
)
|
||||
|
||||
return cls(
|
||||
kv_cache_quant_method=kv_cache_quant_method,
|
||||
@@ -2289,6 +2314,7 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
|
||||
quantized_layers=quantized_layers,
|
||||
fp8_config=fp8_config,
|
||||
nvfp4_config=nvfp4_config,
|
||||
w4a16_nvfp4_config=w4a16_nvfp4_config,
|
||||
)
|
||||
|
||||
def _resolve_quant_algo(self, prefix: str) -> str | None:
|
||||
@@ -2355,6 +2381,8 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
|
||||
return ModelOptFp8LinearMethod(self.fp8_config)
|
||||
if quant_algo == "NVFP4":
|
||||
return ModelOptNvFp4LinearMethod(self.nvfp4_config)
|
||||
if quant_algo == "W4A16_NVFP4":
|
||||
return ModelOptNvFp4W4A16LinearMethod(self.w4a16_nvfp4_config)
|
||||
# Layer not in quantized_layers — leave unquantized
|
||||
return UnquantizedLinearMethod()
|
||||
|
||||
@@ -2369,6 +2397,11 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
|
||||
quant_config=self.nvfp4_config,
|
||||
moe_config=layer.moe_config,
|
||||
)
|
||||
if quant_algo == "W4A16_NVFP4":
|
||||
return ModelOptNvFp4FusedMoE(
|
||||
quant_config=self.w4a16_nvfp4_config,
|
||||
moe_config=layer.moe_config,
|
||||
)
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
Reference in New Issue
Block a user