[Bugfix] Handle ParallelLMHead in compressed-tensors get_quant_method (#37291)
Signed-off-by: Matthias Gehre <matthias.gehre@amd.com> Co-authored-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Michael Goin
parent
246dc7d864
commit
e8b055a5ac
@@ -62,6 +62,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
|
||||
should_ignore_layer,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -179,6 +180,15 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
else:
|
||||
return quant_method
|
||||
|
||||
if isinstance(layer, ParallelLMHead):
|
||||
try:
|
||||
quant_scheme = self.get_scheme(layer=layer, layer_name=prefix)
|
||||
except ValueError:
|
||||
quant_scheme = None
|
||||
if quant_scheme is not None:
|
||||
layer.scheme = quant_scheme
|
||||
return CompressedTensorsLinearMethod(self)
|
||||
|
||||
if isinstance(layer, Attention):
|
||||
return CompressedTensorsKVCacheMethod(self)
|
||||
if isinstance(layer, FusedMoE):
|
||||
|
||||
Reference in New Issue
Block a user