[Bugfix][Quantization] Detect NVFP4 in ModelOpt mixed-precision checkpoints (#56050)
Signed-off-by: Stefano Castagnetta <scastagnetta@nvidia.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
fa212b8dc5
commit
f0b4b36890
@@ -101,6 +101,20 @@ llama3_8b_fp4 = ModelFusionInfo(
|
||||
),
|
||||
)
|
||||
|
||||
# ModelOpt MIXED_PRECISION: FP8 attention projections, NVFP4 MLPs. It resolves to
|
||||
# modelopt_mixed, not modelopt_fp4, and its MLP has no manual act+quant fusion, so
|
||||
# every MLP boundary is left to ActivationQuantFusionPass.
|
||||
qwen3_8_27b_mixed_fp4 = ModelFusionInfo(
|
||||
model_name="nvidia/Qwen3.8-27B-NVFP4",
|
||||
hf_overrides=lambda n_layers: {
|
||||
"text_config": {
|
||||
"num_hidden_layers": n_layers,
|
||||
"layer_types": ["linear_attention"] * (n_layers - 1) + ["full_attention"],
|
||||
}
|
||||
},
|
||||
matches=lambda n_layers: Matches(act_quant_fusion=n_layers),
|
||||
)
|
||||
|
||||
# MoEs cannot do act+quant fusion because those ops are hidden from torch.compile.
|
||||
# MoEs also only expose 1 rms+quant fusion because the quant for up_proj is hidden.
|
||||
# TODO(luka): https://github.com/vllm-project/vllm/issues/31985
|
||||
|
||||
@@ -30,6 +30,7 @@ from .models import (
|
||||
llama3_8b_fp8,
|
||||
llama4_scout_fp4,
|
||||
llama4_scout_fp8,
|
||||
qwen3_8_27b_mixed_fp4,
|
||||
qwen3_a3b_fp8,
|
||||
)
|
||||
|
||||
@@ -202,3 +203,46 @@ def test_tp1_fp4_fusions(
|
||||
compilation_config,
|
||||
matches_check,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_name, matches_fn, model_kwargs, hf_overrides", [qwen3_8_27b_mixed_fp4]
|
||||
)
|
||||
@pytest.mark.parametrize("attn_backend", [FLASHINFER_ATTN])
|
||||
@pytest.mark.parametrize("n_layers", [4])
|
||||
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
|
||||
@pytest.mark.skipif(not is_blackwell(), reason="Blackwell required for fp4")
|
||||
def test_tp1_fp4_mixed_precision_act_fusion(
|
||||
model_name: str,
|
||||
matches_fn: Callable[[int], Matches],
|
||||
model_kwargs: dict,
|
||||
hf_overrides: Callable[[int], dict],
|
||||
attn_backend: AttentionBackendCase,
|
||||
n_layers: int,
|
||||
inductor_graph_partition: bool,
|
||||
run_e2e_fusion_test,
|
||||
):
|
||||
"""Act+quant fusion matches on a MIXED_PRECISION NVFP4 checkpoint."""
|
||||
model_kwargs["hf_overrides"] = hf_overrides(n_layers)
|
||||
model_kwargs["load_format"] = "dummy"
|
||||
model_kwargs["max_model_len"] = 1024
|
||||
model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False}
|
||||
|
||||
compilation_config = dict(
|
||||
use_inductor_graph_partition=inductor_graph_partition,
|
||||
pass_config=PassConfig(
|
||||
fuse_norm_quant=False,
|
||||
fuse_act_quant=True,
|
||||
fuse_attn_quant=False,
|
||||
enable_qk_norm_rope_fusion=False,
|
||||
),
|
||||
)
|
||||
|
||||
run_e2e_fusion_test(
|
||||
model_name,
|
||||
matches_fn(n_layers),
|
||||
model_kwargs,
|
||||
attn_backend,
|
||||
compilation_config,
|
||||
["act_quant_fusion"],
|
||||
)
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""NVFP4 detection for ModelOpt MIXED_PRECISION checkpoints.
|
||||
|
||||
A checkpoint that quantizes some layers to NVFP4 and others to FP8 declares
|
||||
``quant_algo: MIXED_PRECISION`` and resolves to ``modelopt_mixed``, so NVFP4
|
||||
has to be detected from the per-layer algorithms.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config.model import _modelopt_mixed_has_nvfp4
|
||||
|
||||
|
||||
def _quantized_layers(*algos: str) -> dict:
|
||||
return {
|
||||
"quantized_layers": {
|
||||
f"model.layers.{i}.proj": {"quant_algo": algo}
|
||||
for i, algo in enumerate(algos)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def test_detects_nvfp4_alongside_another_algorithm():
|
||||
"""The shipped shape: FP8 attention projections, NVFP4 MLP."""
|
||||
assert _modelopt_mixed_has_nvfp4(_quantized_layers("FP8", "NVFP4", "FP8"))
|
||||
|
||||
|
||||
def test_ignores_a_checkpoint_with_no_nvfp4_layer():
|
||||
assert not _modelopt_mixed_has_nvfp4(_quantized_layers("FP8", "MXFP8"))
|
||||
|
||||
|
||||
def test_weight_only_nvfp4_does_not_count():
|
||||
"""W4A16_NVFP4 leaves activations in high precision, so nothing to fuse."""
|
||||
assert not _modelopt_mixed_has_nvfp4(_quantized_layers("FP8", "W4A16_NVFP4"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"quant_config",
|
||||
[None, {}, {"quantized_layers": None}, {"quantized_layers": ["NVFP4"]}],
|
||||
)
|
||||
def test_absent_or_malformed_config_is_not_nvfp4(quant_config):
|
||||
assert not _modelopt_mixed_has_nvfp4(quant_config)
|
||||
+24
-1
@@ -124,6 +124,22 @@ _RUNNER_CONVERTS: dict[RunnerType, list[ConvertType]] = {
|
||||
"draft": [],
|
||||
}
|
||||
|
||||
|
||||
def _modelopt_mixed_has_nvfp4(quant_config: dict[str, Any] | None) -> bool:
|
||||
"""Whether a ModelOpt MIXED_PRECISION checkpoint quantizes any layer to NVFP4.
|
||||
|
||||
``W4A16_NVFP4`` is excluded on purpose: it quantizes weights only, so there
|
||||
is no activation quantization for the fusion passes to act on.
|
||||
"""
|
||||
layers = (quant_config or {}).get("quantized_layers")
|
||||
if not isinstance(layers, dict):
|
||||
return False
|
||||
return any(
|
||||
isinstance(info, dict) and info.get("quant_algo") == "NVFP4"
|
||||
for info in layers.values()
|
||||
)
|
||||
|
||||
|
||||
AttnTypeStr = Literal[
|
||||
"decoder", "encoder", "encoder_only", "encoder_decoder", "attention_free", "hybrid"
|
||||
]
|
||||
@@ -2249,13 +2265,20 @@ class ModelConfig:
|
||||
return getattr(self.hf_config, "quantization_config", None) is not None
|
||||
|
||||
def is_nvfp4_quantized(self) -> bool:
|
||||
quant_config = self.model_arch_config.quantization_config
|
||||
|
||||
# ModelOpt NVFP4 checkpoints resolve to modelopt_fp4 quantization method
|
||||
if self.quantization in ("modelopt_fp4",):
|
||||
return True
|
||||
|
||||
# A checkpoint mixing NVFP4 with another algorithm declares
|
||||
# quant_algo MIXED_PRECISION and resolves to modelopt_mixed, so the
|
||||
# per-layer algorithms decide.
|
||||
if self.quantization == "modelopt_mixed":
|
||||
return _modelopt_mixed_has_nvfp4(quant_config)
|
||||
|
||||
# For Compressed Tensors we look for `"format": "nvfp4-pack-quantized"`
|
||||
# in the quantization config
|
||||
quant_config = self.model_arch_config.quantization_config
|
||||
return (
|
||||
self.quantization == "compressed-tensors"
|
||||
and quant_config is not None
|
||||
|
||||
Reference in New Issue
Block a user