[Bugfix] Fix Qwen4Exp PLE embedding rejecting INC (AutoRound) checkpoints (#59990)
Signed-off-by: Vadim Gimpelson <vadim.gimpelson@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude
parent
49e0f47978
commit
4f52fa35ef
@@ -20,6 +20,7 @@ from vllm.config import SpeculativeConfig, VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.model_executor.layers.quantization.inc import INCConfig
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptMixedPrecisionConfig,
|
||||
ModelOptNvFp4Config,
|
||||
@@ -566,6 +567,24 @@ def test_ple_embedding_respects_modelopt_exclusion() -> None:
|
||||
)
|
||||
|
||||
|
||||
def test_ple_embedding_respects_inc_layer_config() -> None:
|
||||
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
|
||||
quant_config = INCConfig(
|
||||
weight_bits=4,
|
||||
group_size=128,
|
||||
block_name_to_quantize="model.layers",
|
||||
extra_config={".*ple.*": {"bits": 16, "data_type": "float"}},
|
||||
)
|
||||
assert isinstance(
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix),
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
|
||||
quant_config.extra_config = None
|
||||
with pytest.raises(NotImplementedError, match="INCConfig"):
|
||||
Qwen4ExpPLEEmbeddingMethod.from_quant_config(quant_config, prefix)
|
||||
|
||||
|
||||
def test_ple_embedding_dtype_overrides_modelopt_exclusion() -> None:
|
||||
prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"
|
||||
quant_config = ModelOptNvFp4Config(exclude_modules=[prefix])
|
||||
|
||||
@@ -27,6 +27,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tenso
|
||||
CompressedTensorsConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.model_executor.layers.quantization.inc import INCConfig
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptMixedPrecisionConfig,
|
||||
ModelOptQuantConfigBase,
|
||||
@@ -190,6 +191,11 @@ class Qwen4ExpPLEEmbeddingMethod(QuantizeMethodBase):
|
||||
and quant_config.get_scheme_dict(None, layer_name=prefix) is None
|
||||
):
|
||||
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
||||
if (
|
||||
isinstance(quant_config, INCConfig)
|
||||
and not quant_config.config_parser.resolve(None, prefix).quantized
|
||||
):
|
||||
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
||||
if not isinstance(quant_config, Fp8Config):
|
||||
raise NotImplementedError(
|
||||
"Qwen4Exp PLE embedding does not support quantization config "
|
||||
|
||||
Reference in New Issue
Block a user