[Quantization] Add ModelOpt IQ2_XS, IQ2_XXS, and Q8_0 support via b12x (#59604)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Signed-off-by: Luke Alonso <lalonso@gmail.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: Luke Alonso <lalonso@gmail.com>
This commit is contained in:
co-authored by
Andrii Skliar
Luke Alonso
parent
ac71b17517
commit
6849ca696a
@@ -99,7 +99,10 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
|
||||
requantize_with_max_scale,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
BlockQuantScaleParameter,
|
||||
ChannelQuantScaleParameter,
|
||||
@@ -111,6 +114,7 @@ from vllm.model_executor.utils import (
|
||||
replace_parameter,
|
||||
set_weight_attrs,
|
||||
)
|
||||
from vllm.utils.b12x import B12X_BLOCK_CODECS
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -1559,6 +1563,19 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
|
||||
super().__init__(exclude_modules)
|
||||
self.kv_cache_quant_method = kv_cache_quant_method
|
||||
self.quantized_layers = quantized_layers
|
||||
for prefix, recipe in quantized_layers.items():
|
||||
codec = recipe.get("quant_algo", "").lower()
|
||||
if codec in B12X_BLOCK_CODECS and any(
|
||||
recipe.get(key) != value
|
||||
for key, value in {
|
||||
"group_size": B12X_BLOCK_CODECS[codec][0],
|
||||
"block_payload_bytes": B12X_BLOCK_CODECS[codec][1],
|
||||
"packing": "ggml",
|
||||
}.items()
|
||||
):
|
||||
raise ValueError(
|
||||
f"unsupported {codec.upper()} block contract for {prefix}: {recipe}"
|
||||
)
|
||||
self.fp8_config = fp8_config
|
||||
self.nvfp4_config = nvfp4_config
|
||||
self.w4a16_nvfp4_config = w4a16_nvfp4_config
|
||||
@@ -1811,6 +1828,22 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
|
||||
|
||||
quant_algo = self._resolve_quant_algo(prefix)
|
||||
|
||||
if quant_algo is not None and quant_algo.lower() in B12X_BLOCK_CODECS:
|
||||
from .modelopt_block_quant import (
|
||||
ModelOptBlockQuantLinearMethod,
|
||||
ModelOptBlockQuantMoEMethod,
|
||||
)
|
||||
|
||||
if isinstance(layer, RoutedExperts):
|
||||
return ModelOptBlockQuantMoEMethod(
|
||||
layer.moe_config, codec=quant_algo.lower()
|
||||
)
|
||||
if isinstance(layer, (LinearBase, VocabParallelEmbedding)):
|
||||
return ModelOptBlockQuantLinearMethod(codec=quant_algo.lower())
|
||||
raise ValueError(
|
||||
f"{quant_algo} requires a linear, embedding or routed experts"
|
||||
)
|
||||
|
||||
if isinstance(layer, (LinearBase, ParallelLMHead)):
|
||||
# Per-prefix algo -> its sub-config, then the generic linear method.
|
||||
# resolve() reads group_size off whichever sub-config it is handed.
|
||||
|
||||
Reference in New Issue
Block a user