diff --git a/tests/compile/fusions_e2e/models.py b/tests/compile/fusions_e2e/models.py index eadf1739fd..082ab0bf52 100644 --- a/tests/compile/fusions_e2e/models.py +++ b/tests/compile/fusions_e2e/models.py @@ -101,6 +101,20 @@ llama3_8b_fp4 = ModelFusionInfo( ), ) +# ModelOpt MIXED_PRECISION: FP8 attention projections, NVFP4 MLPs. It resolves to +# modelopt_mixed, not modelopt_fp4, and its MLP has no manual act+quant fusion, so +# every MLP boundary is left to ActivationQuantFusionPass. +qwen3_8_27b_mixed_fp4 = ModelFusionInfo( + model_name="nvidia/Qwen3.8-27B-NVFP4", + hf_overrides=lambda n_layers: { + "text_config": { + "num_hidden_layers": n_layers, + "layer_types": ["linear_attention"] * (n_layers - 1) + ["full_attention"], + } + }, + matches=lambda n_layers: Matches(act_quant_fusion=n_layers), +) + # MoEs cannot do act+quant fusion because those ops are hidden from torch.compile. # MoEs also only expose 1 rms+quant fusion because the quant for up_proj is hidden. # TODO(luka): https://github.com/vllm-project/vllm/issues/31985 diff --git a/tests/compile/fusions_e2e/test_tp1_quant.py b/tests/compile/fusions_e2e/test_tp1_quant.py index fa1af8e1c6..cec0cbecae 100644 --- a/tests/compile/fusions_e2e/test_tp1_quant.py +++ b/tests/compile/fusions_e2e/test_tp1_quant.py @@ -30,6 +30,7 @@ from .models import ( llama3_8b_fp8, llama4_scout_fp4, llama4_scout_fp8, + qwen3_8_27b_mixed_fp4, qwen3_a3b_fp8, ) @@ -202,3 +203,46 @@ def test_tp1_fp4_fusions( compilation_config, matches_check, ) + + +@pytest.mark.parametrize( + "model_name, matches_fn, model_kwargs, hf_overrides", [qwen3_8_27b_mixed_fp4] +) +@pytest.mark.parametrize("attn_backend", [FLASHINFER_ATTN]) +@pytest.mark.parametrize("n_layers", [4]) +@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION) +@pytest.mark.skipif(not is_blackwell(), reason="Blackwell required for fp4") +def test_tp1_fp4_mixed_precision_act_fusion( + model_name: str, + matches_fn: Callable[[int], Matches], + model_kwargs: dict, + hf_overrides: Callable[[int], dict], + attn_backend: AttentionBackendCase, + n_layers: int, + inductor_graph_partition: bool, + run_e2e_fusion_test, +): + """Act+quant fusion matches on a MIXED_PRECISION NVFP4 checkpoint.""" + model_kwargs["hf_overrides"] = hf_overrides(n_layers) + model_kwargs["load_format"] = "dummy" + model_kwargs["max_model_len"] = 1024 + model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False} + + compilation_config = dict( + use_inductor_graph_partition=inductor_graph_partition, + pass_config=PassConfig( + fuse_norm_quant=False, + fuse_act_quant=True, + fuse_attn_quant=False, + enable_qk_norm_rope_fusion=False, + ), + ) + + run_e2e_fusion_test( + model_name, + matches_fn(n_layers), + model_kwargs, + attn_backend, + compilation_config, + ["act_quant_fusion"], + ) diff --git a/tests/config/test_modelopt_mixed_nvfp4.py b/tests/config/test_modelopt_mixed_nvfp4.py new file mode 100644 index 0000000000..b9a890144a --- /dev/null +++ b/tests/config/test_modelopt_mixed_nvfp4.py @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""NVFP4 detection for ModelOpt MIXED_PRECISION checkpoints. + +A checkpoint that quantizes some layers to NVFP4 and others to FP8 declares +``quant_algo: MIXED_PRECISION`` and resolves to ``modelopt_mixed``, so NVFP4 +has to be detected from the per-layer algorithms. +""" + +import pytest + +from vllm.config.model import _modelopt_mixed_has_nvfp4 + + +def _quantized_layers(*algos: str) -> dict: + return { + "quantized_layers": { + f"model.layers.{i}.proj": {"quant_algo": algo} + for i, algo in enumerate(algos) + } + } + + +def test_detects_nvfp4_alongside_another_algorithm(): + """The shipped shape: FP8 attention projections, NVFP4 MLP.""" + assert _modelopt_mixed_has_nvfp4(_quantized_layers("FP8", "NVFP4", "FP8")) + + +def test_ignores_a_checkpoint_with_no_nvfp4_layer(): + assert not _modelopt_mixed_has_nvfp4(_quantized_layers("FP8", "MXFP8")) + + +def test_weight_only_nvfp4_does_not_count(): + """W4A16_NVFP4 leaves activations in high precision, so nothing to fuse.""" + assert not _modelopt_mixed_has_nvfp4(_quantized_layers("FP8", "W4A16_NVFP4")) + + +@pytest.mark.parametrize( + "quant_config", + [None, {}, {"quantized_layers": None}, {"quantized_layers": ["NVFP4"]}], +) +def test_absent_or_malformed_config_is_not_nvfp4(quant_config): + assert not _modelopt_mixed_has_nvfp4(quant_config) diff --git a/vllm/config/model.py b/vllm/config/model.py index 4de030e8f3..75ca774fa1 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -124,6 +124,22 @@ _RUNNER_CONVERTS: dict[RunnerType, list[ConvertType]] = { "draft": [], } + +def _modelopt_mixed_has_nvfp4(quant_config: dict[str, Any] | None) -> bool: + """Whether a ModelOpt MIXED_PRECISION checkpoint quantizes any layer to NVFP4. + + ``W4A16_NVFP4`` is excluded on purpose: it quantizes weights only, so there + is no activation quantization for the fusion passes to act on. + """ + layers = (quant_config or {}).get("quantized_layers") + if not isinstance(layers, dict): + return False + return any( + isinstance(info, dict) and info.get("quant_algo") == "NVFP4" + for info in layers.values() + ) + + AttnTypeStr = Literal[ "decoder", "encoder", "encoder_only", "encoder_decoder", "attention_free", "hybrid" ] @@ -2249,13 +2265,20 @@ class ModelConfig: return getattr(self.hf_config, "quantization_config", None) is not None def is_nvfp4_quantized(self) -> bool: + quant_config = self.model_arch_config.quantization_config + # ModelOpt NVFP4 checkpoints resolve to modelopt_fp4 quantization method if self.quantization in ("modelopt_fp4",): return True + # A checkpoint mixing NVFP4 with another algorithm declares + # quant_algo MIXED_PRECISION and resolves to modelopt_mixed, so the + # per-layer algorithms decide. + if self.quantization == "modelopt_mixed": + return _modelopt_mixed_has_nvfp4(quant_config) + # For Compressed Tensors we look for `"format": "nvfp4-pack-quantized"` # in the quantization config - quant_config = self.model_arch_config.quantization_config return ( self.quantization == "compressed-tensors" and quant_config is not None