[Bugfix][Quantization] Detect NVFP4 in ModelOpt mixed-precision checkpoints (#56050)

Signed-off-by: Stefano Castagnetta <scastagnetta@nvidia.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Stefano Castagnetta
2026-10-02 12:30:22 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent fa212b8dc5
commit f0b4b36890
4 changed files with 125 additions and 1 deletions
+14
View File
@@ -101,6 +101,20 @@ llama3_8b_fp4 = ModelFusionInfo(
),
)
# ModelOpt MIXED_PRECISION: FP8 attention projections, NVFP4 MLPs. It resolves to
# modelopt_mixed, not modelopt_fp4, and its MLP has no manual act+quant fusion, so
# every MLP boundary is left to ActivationQuantFusionPass.
qwen3_8_27b_mixed_fp4 = ModelFusionInfo(
model_name="nvidia/Qwen3.8-27B-NVFP4",
hf_overrides=lambda n_layers: {
"text_config": {
"num_hidden_layers": n_layers,
"layer_types": ["linear_attention"] * (n_layers - 1) + ["full_attention"],
}
},
matches=lambda n_layers: Matches(act_quant_fusion=n_layers),
)
# MoEs cannot do act+quant fusion because those ops are hidden from torch.compile.
# MoEs also only expose 1 rms+quant fusion because the quant for up_proj is hidden.
# TODO(luka): https://github.com/vllm-project/vllm/issues/31985
@@ -30,6 +30,7 @@ from .models import (
llama3_8b_fp8,
llama4_scout_fp4,
llama4_scout_fp8,
qwen3_8_27b_mixed_fp4,
qwen3_a3b_fp8,
)
@@ -202,3 +203,46 @@ def test_tp1_fp4_fusions(
compilation_config,
matches_check,
)
@pytest.mark.parametrize(
"model_name, matches_fn, model_kwargs, hf_overrides", [qwen3_8_27b_mixed_fp4]
)
@pytest.mark.parametrize("attn_backend", [FLASHINFER_ATTN])
@pytest.mark.parametrize("n_layers", [4])
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
@pytest.mark.skipif(not is_blackwell(), reason="Blackwell required for fp4")
def test_tp1_fp4_mixed_precision_act_fusion(
model_name: str,
matches_fn: Callable[[int], Matches],
model_kwargs: dict,
hf_overrides: Callable[[int], dict],
attn_backend: AttentionBackendCase,
n_layers: int,
inductor_graph_partition: bool,
run_e2e_fusion_test,
):
"""Act+quant fusion matches on a MIXED_PRECISION NVFP4 checkpoint."""
model_kwargs["hf_overrides"] = hf_overrides(n_layers)
model_kwargs["load_format"] = "dummy"
model_kwargs["max_model_len"] = 1024
model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False}
compilation_config = dict(
use_inductor_graph_partition=inductor_graph_partition,
pass_config=PassConfig(
fuse_norm_quant=False,
fuse_act_quant=True,
fuse_attn_quant=False,
enable_qk_norm_rope_fusion=False,
),
)
run_e2e_fusion_test(
model_name,
matches_fn(n_layers),
model_kwargs,
attn_backend,
compilation_config,
["act_quant_fusion"],
)