[Quantization] Rework quantization_config to use QuantKey and allow for activation override (#41566)

This commit is contained in:
Michael Goin
2026-05-13 16:58:32 -04:00
committed by GitHub
parent cca32d55a2
commit 8efd508204
16 changed files with 621 additions and 322 deletions
@@ -300,12 +300,10 @@ def get_quant_config(
)
if hf_quant_config is not None:
if model_config.quantization_config is not None:
raise ValueError(
"Setting `quantization_config` for online "
"quantization when the model checkpoint already "
"has a `quantization_config` is not supported"
)
# `model_config.quantization_config` may be set alongside a checkpoint
# quant config: the checkpoint determines `quant_cls`, and the user's
# QuantizationConfigArgs is consulted by individual quant methods
# (e.g. for activation overrides via the MXFP4 oracle).
# For modelopt_mixed, config.json's quantization_config may or may
# not contain the per-layer quantized_layers map. Newer checkpoints
@@ -330,12 +328,6 @@ def get_quant_config(
quantization_config_file = hf_overrides.get("quantization_config_file", None)
if quantization_config_file is not None:
if hasattr(quant_cls, "from_config_file"):
if model_config.quantization_config is not None:
raise ValueError(
"Setting `quantization_config` for online "
"quantization when the model checkpoint already "
"has a `quantization_config` is not supported"
)
return quant_cls.from_config_file(quantization_config_file)
else:
raise NotImplementedError(
@@ -346,12 +338,6 @@ def get_quant_config(
quantization_config_json = hf_overrides.get("quantization_config_dict_json", None)
if quantization_config_json is not None:
if hasattr(quant_cls, "from_config_dict_json"):
if model_config.quantization_config is not None:
raise ValueError(
"Setting `quantization_config` for online "
"quantization when the model checkpoint already "
"has a `quantization_config` is not supported"
)
return quant_cls.from_config_dict_json(quantization_config_json)
else:
raise NotImplementedError(
@@ -360,17 +346,15 @@ def get_quant_config(
f"{quant_cls}"
)
# Online quantization doesn't read from checkpoint configs — it quantizes
# Online quantization doesn't read from checkpoint configs - it quantizes
# fp16/bf16 weights on the fly during loading.
if model_config.quantization_config is not None:
from vllm.config.quantization import OnlineQuantizationConfigArgs
from vllm.config.quantization import QuantizationConfigArgs
from vllm.model_executor.layers.quantization.online.base import (
OnlineQuantizationConfig,
)
assert isinstance(
model_config.quantization_config, OnlineQuantizationConfigArgs
)
assert isinstance(model_config.quantization_config, QuantizationConfigArgs)
return OnlineQuantizationConfig(args=model_config.quantization_config)
# Inflight BNB quantization