[Bugfix] Handle ParallelLMHead in compressed-tensors get_quant_method (#37291)

Signed-off-by: Matthias Gehre <matthias.gehre@amd.com>
Co-authored-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
Matthias Gehre
2026-03-30 07:30:52 -07:00
committed by GitHub
co-authored by Michael Goin
parent 246dc7d864
commit e8b055a5ac
2 changed files with 88 additions and 1 deletions
@@ -62,6 +62,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
should_ignore_layer,
)
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.platforms import current_platform
if TYPE_CHECKING:
@@ -179,6 +180,15 @@ class CompressedTensorsConfig(QuantizationConfig):
else:
return quant_method
if isinstance(layer, ParallelLMHead):
try:
quant_scheme = self.get_scheme(layer=layer, layer_name=prefix)
except ValueError:
quant_scheme = None
if quant_scheme is not None:
layer.scheme = quant_scheme
return CompressedTensorsLinearMethod(self)
if isinstance(layer, Attention):
return CompressedTensorsKVCacheMethod(self)
if isinstance(layer, FusedMoE):