Add LM head quantization support for ModelOpt (#42124)

Signed-off-by: weimingc <17592131+meenchen@users.noreply.github.com>
This commit is contained in:
Wei-Ming Chen
2026-05-26 09:21:05 -07:00
committed by GitHub
parent c8414a8271
commit 6f5b533241
8 changed files with 220 additions and 5 deletions
@@ -81,6 +81,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
requantize_with_max_scale,
)
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.parameter import (
BlockQuantScaleParameter,
ChannelQuantScaleParameter,
@@ -182,7 +183,7 @@ class ModelOptQuantConfigBase(QuantizationConfig):
# handle exclusion
if self.is_layer_excluded(prefix):
if isinstance(layer, LinearBase):
if isinstance(layer, (LinearBase, ParallelLMHead)):
return UnquantizedLinearMethod()
return None
@@ -195,7 +196,7 @@ class ModelOptQuantConfigBase(QuantizationConfig):
return UnquantizedLinearMethod()
# now, the layer is quantized, handle it here
if isinstance(layer, LinearBase):
if isinstance(layer, (LinearBase, ParallelLMHead)):
quant_method = self.LinearMethodCls(self)
if getattr(quant_method, "backend", "") == "marlin":
quant_method.marlin_input_dtype = get_marlin_input_dtype(prefix)
@@ -2371,13 +2372,13 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
# Excluded layers
if self.is_layer_excluded(prefix):
if isinstance(layer, LinearBase):
if isinstance(layer, (LinearBase, ParallelLMHead)):
return UnquantizedLinearMethod()
return None
quant_algo = self._resolve_quant_algo(prefix)
if isinstance(layer, LinearBase):
if isinstance(layer, (LinearBase, ParallelLMHead)):
if quant_algo == "FP8":
return ModelOptFp8LinearMethod(self.fp8_config)
if quant_algo == "NVFP4":
@@ -290,6 +290,7 @@ class VocabParallelEmbedding(PluggableLayer):
if params_dtype is None:
params_dtype = torch.get_default_dtype()
self.params_dtype = params_dtype
# Divide the weight matrix along the vocabulary dimension.
self.num_added_embeddings = self.num_embeddings - self.org_vocab_size
self.num_embeddings_per_partition = divide(
@@ -438,6 +439,12 @@ class VocabParallelEmbedding(PluggableLayer):
# If parameter does not have output dim, then it should
# be copied onto all gpus (e.g. g_idx for act_order gptq).
if output_dim is None:
if (
loaded_weight.ndim == 0
and param.data.ndim == 1
and param.data.numel() == 1
):
loaded_weight = loaded_weight.reshape(1)
assert param.data.shape == loaded_weight.shape
param.data.copy_(loaded_weight)
return
+1
View File
@@ -875,6 +875,7 @@ class NemotronHForCausalLM(
self.lm_head = ParallelLMHead(
config.vocab_size,
config.hidden_size,
quant_config=self.quant_config,
prefix=maybe_prefix(prefix, "lm_head"),
)
+1
View File
@@ -477,6 +477,7 @@ class Qwen3_5ForCausalLMBase(
self.lm_head = ParallelLMHead(
config.vocab_size,
config.hidden_size,
quant_config=self.quant_config,
prefix=maybe_prefix(prefix, "lm_head"),
)
else:
@@ -381,6 +381,7 @@ class Qwen3_5MTP(nn.Module, SupportsMultiModal):
self.lm_head = ParallelLMHead(
config.vocab_size,
config.hidden_size,
quant_config=self.quant_config,
prefix=maybe_prefix(prefix, "lm_head"),
)
else: