[BugFix] Fix ModelOpt mixed-precision quantization for sparse quantized_layers configs. (#47318)

Signed-off-by: Daniel Afrimi <dafrimi@nvidia.com>
Signed-off-by: <dafrimi@nvidia.com>
This commit is contained in:
danielafrimi
2026-07-07 11:45:13 +00:00
committed by GitHub
parent 0ed05b6f82
commit 0a2965b1b3
2 changed files with 66 additions and 10 deletions
@@ -2461,16 +2461,29 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
if key.startswith(parent_dot):
return info["quant_algo"].upper()
# 4. Parent-prefix fallback for fused projections (qkv_proj, gate_up_proj).
for candidate in self._quantized_layer_prefix_candidates(prefix):
parent_dot = candidate.rsplit(".", 1)[0] + "."
algos = {
info["quant_algo"].upper()
for key, info in self.quantized_layers.items()
if key.startswith(parent_dot) and "." not in key[len(parent_dot) :]
}
if len(algos) == 1:
return algos.pop()
# 4. Parent-prefix fallback for fused projections whose config lists
# shard names instead of vLLM's packed module name.
fused_projection_shards = {
"qkv_proj": ("q_proj", "k_proj", "v_proj"),
"gate_up_proj": ("gate_proj", "up_proj"),
}
shard_names = fused_projection_shards.get(proj_name)
if shard_names is not None:
for candidate in self._quantized_layer_prefix_candidates(prefix):
parent_dot = candidate.rsplit(".", 1)[0] + "."
shard_algos: set[str] = set()
for shard_name in shard_names:
shard_prefix = f"{parent_dot}{shard_name}"
if shard_prefix in self.quantized_layers:
algo = self.quantized_layers[shard_prefix]["quant_algo"].upper()
shard_algos.add(algo)
if len(shard_algos) == 1:
return shard_algos.pop()
if len(shard_algos) > 1:
raise ValueError(
f"Mixed quant_algo within fused layer {prefix}: "
f"{shard_algos}. All shards must use the same quantization."
)
return None