From 0a2965b1b35f34fde29c9a653696bac0ef1e499d Mon Sep 17 00:00:00 2001 From: danielafrimi <45691845+danielafrimi@users.noreply.github.com> Date: Tue, 7 Jul 2026 14:45:13 +0300 Subject: [PATCH] [BugFix] Fix ModelOpt mixed-precision quantization for sparse `quantized_layers` configs. (#47318) Signed-off-by: Daniel Afrimi Signed-off-by: --- tests/quantization/test_modelopt.py | 43 +++++++++++++++++++ .../layers/quantization/modelopt.py | 33 +++++++++----- 2 files changed, 66 insertions(+), 10 deletions(-) diff --git a/tests/quantization/test_modelopt.py b/tests/quantization/test_modelopt.py index b43f4f8d35..f10b3c9a55 100644 --- a/tests/quantization/test_modelopt.py +++ b/tests/quantization/test_modelopt.py @@ -133,6 +133,49 @@ def test_modelopt_mixed_precision_quantizes_parallel_lm_head(): assert isinstance(method, ModelOptNvFp4LinearMethod) +def test_modelopt_mixed_precision_infers_fused_gate_up_projection(): + from vllm.model_executor.layers.linear import LinearBase + + config = _mixed_precision_config( + { + "model.layers.0.mlp.gate_proj": {"quant_algo": "NVFP4"}, + "model.layers.0.mlp.up_proj": {"quant_algo": "NVFP4"}, + } + ) + + fake_layer = MagicMock(spec=LinearBase) + with patch( + "vllm.model_executor.layers.quantization.modelopt.init_nvfp4_linear_kernel" + ): + method = config.get_quant_method(fake_layer, "model.layers.0.mlp.gate_up_proj") + + assert isinstance(method, ModelOptNvFp4LinearMethod) + + +@pytest.mark.parametrize( + ("quantized_prefix", "missing_prefix"), + [ + ("model.layers.0.mlp.gate_proj", "model.layers.0.mlp.down_proj"), + ("model.layers.0.self_attn.o_proj", "model.layers.0.self_attn.qkv_proj"), + ], +) +def test_modelopt_mixed_precision_does_not_infer_missing_sibling_linear( + quantized_prefix, missing_prefix +): + from vllm.model_executor.layers.linear import LinearBase + + config = _mixed_precision_config( + { + quantized_prefix: {"quant_algo": "NVFP4"}, + } + ) + + fake_layer = MagicMock(spec=LinearBase) + method = config.get_quant_method(fake_layer, missing_prefix) + + assert isinstance(method, UnquantizedLinearMethod) + + def test_vocab_parallel_embedding_weight_loader_accepts_scalar_scale(): holder = Mock() scale = torch.nn.Parameter(torch.empty(1)) diff --git a/vllm/model_executor/layers/quantization/modelopt.py b/vllm/model_executor/layers/quantization/modelopt.py index 352bf90ea7..7df8178ca7 100644 --- a/vllm/model_executor/layers/quantization/modelopt.py +++ b/vllm/model_executor/layers/quantization/modelopt.py @@ -2461,16 +2461,29 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase): if key.startswith(parent_dot): return info["quant_algo"].upper() - # 4. Parent-prefix fallback for fused projections (qkv_proj, gate_up_proj). - for candidate in self._quantized_layer_prefix_candidates(prefix): - parent_dot = candidate.rsplit(".", 1)[0] + "." - algos = { - info["quant_algo"].upper() - for key, info in self.quantized_layers.items() - if key.startswith(parent_dot) and "." not in key[len(parent_dot) :] - } - if len(algos) == 1: - return algos.pop() + # 4. Parent-prefix fallback for fused projections whose config lists + # shard names instead of vLLM's packed module name. + fused_projection_shards = { + "qkv_proj": ("q_proj", "k_proj", "v_proj"), + "gate_up_proj": ("gate_proj", "up_proj"), + } + shard_names = fused_projection_shards.get(proj_name) + if shard_names is not None: + for candidate in self._quantized_layer_prefix_candidates(prefix): + parent_dot = candidate.rsplit(".", 1)[0] + "." + shard_algos: set[str] = set() + for shard_name in shard_names: + shard_prefix = f"{parent_dot}{shard_name}" + if shard_prefix in self.quantized_layers: + algo = self.quantized_layers[shard_prefix]["quant_algo"].upper() + shard_algos.add(algo) + if len(shard_algos) == 1: + return shard_algos.pop() + if len(shard_algos) > 1: + raise ValueError( + f"Mixed quant_algo within fused layer {prefix}: " + f"{shard_algos}. All shards must use the same quantization." + ) return None