Add support for unquantized ngram in CT format (#59431)
This commit is contained in:
@@ -23,6 +23,9 @@ from vllm.model_executor.layers.quantization.base_config import (
|
|||||||
QuantizationConfig,
|
QuantizationConfig,
|
||||||
QuantizeMethodBase,
|
QuantizeMethodBase,
|
||||||
)
|
)
|
||||||
|
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
|
||||||
|
CompressedTensorsConfig,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||||
from vllm.model_executor.layers.quantization.modelopt import (
|
from vllm.model_executor.layers.quantization.modelopt import (
|
||||||
ModelOptMixedPrecisionConfig,
|
ModelOptMixedPrecisionConfig,
|
||||||
@@ -182,6 +185,11 @@ class Qwen4ExpPLEEmbeddingMethod(QuantizeMethodBase):
|
|||||||
quant_config, ModelOptQuantConfigBase
|
quant_config, ModelOptQuantConfigBase
|
||||||
) and quant_config.is_layer_excluded(prefix):
|
) and quant_config.is_layer_excluded(prefix):
|
||||||
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
||||||
|
if (
|
||||||
|
isinstance(quant_config, CompressedTensorsConfig)
|
||||||
|
and quant_config.get_scheme_dict(None, layer_name=prefix) is None
|
||||||
|
):
|
||||||
|
return Qwen4ExpPLEUnquantizedEmbeddingMethod()
|
||||||
if not isinstance(quant_config, Fp8Config):
|
if not isinstance(quant_config, Fp8Config):
|
||||||
raise NotImplementedError(
|
raise NotImplementedError(
|
||||||
"Qwen4Exp PLE embedding does not support quantization config "
|
"Qwen4Exp PLE embedding does not support quantization config "
|
||||||
|
|||||||
Reference in New Issue
Block a user