[Quantization] Add ModelOpt IQ2_XS, IQ2_XXS, and Q8_0 support via b12x (#59604)

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
Signed-off-by: Luke Alonso <lalonso@gmail.com>
Co-authored-by: Andrii Skliar <askliar@nvidia.com>
Co-authored-by: Luke Alonso <lalonso@gmail.com>
This commit is contained in:
Andrii Skliar
2026-10-13 09:06:33 -07:00
committed by GitHub
co-authored by Andrii Skliar Luke Alonso
parent ac71b17517
commit 6849ca696a
15 changed files with 1238 additions and 25 deletions
@@ -99,7 +99,10 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
requantize_with_max_scale,
)
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.parameter import (
BlockQuantScaleParameter,
ChannelQuantScaleParameter,
@@ -111,6 +114,7 @@ from vllm.model_executor.utils import (
replace_parameter,
set_weight_attrs,
)
from vllm.utils.b12x import B12X_BLOCK_CODECS
from vllm.utils.math_utils import cdiv
if TYPE_CHECKING:
@@ -1559,6 +1563,19 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
super().__init__(exclude_modules)
self.kv_cache_quant_method = kv_cache_quant_method
self.quantized_layers = quantized_layers
for prefix, recipe in quantized_layers.items():
codec = recipe.get("quant_algo", "").lower()
if codec in B12X_BLOCK_CODECS and any(
recipe.get(key) != value
for key, value in {
"group_size": B12X_BLOCK_CODECS[codec][0],
"block_payload_bytes": B12X_BLOCK_CODECS[codec][1],
"packing": "ggml",
}.items()
):
raise ValueError(
f"unsupported {codec.upper()} block contract for {prefix}: {recipe}"
)
self.fp8_config = fp8_config
self.nvfp4_config = nvfp4_config
self.w4a16_nvfp4_config = w4a16_nvfp4_config
@@ -1811,6 +1828,22 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
quant_algo = self._resolve_quant_algo(prefix)
if quant_algo is not None and quant_algo.lower() in B12X_BLOCK_CODECS:
from .modelopt_block_quant import (
ModelOptBlockQuantLinearMethod,
ModelOptBlockQuantMoEMethod,
)
if isinstance(layer, RoutedExperts):
return ModelOptBlockQuantMoEMethod(
layer.moe_config, codec=quant_algo.lower()
)
if isinstance(layer, (LinearBase, VocabParallelEmbedding)):
return ModelOptBlockQuantLinearMethod(codec=quant_algo.lower())
raise ValueError(
f"{quant_algo} requires a linear, embedding or routed experts"
)
if isinstance(layer, (LinearBase, ParallelLMHead)):
# Per-prefix algo -> its sub-config, then the generic linear method.
# resolve() reads group_size off whichever sub-config it is handed.