Add LM head quantization support for ModelOpt (#42124)

Signed-off-by: weimingc <17592131+meenchen@users.noreply.github.com>
This commit is contained in:
Wei-Ming Chen
2026-05-26 09:21:05 -07:00
committed by GitHub
parent c8414a8271
commit 6f5b533241
8 changed files with 220 additions and 5 deletions
@@ -290,6 +290,7 @@ class VocabParallelEmbedding(PluggableLayer):
if params_dtype is None:
params_dtype = torch.get_default_dtype()
self.params_dtype = params_dtype
# Divide the weight matrix along the vocabulary dimension.
self.num_added_embeddings = self.num_embeddings - self.org_vocab_size
self.num_embeddings_per_partition = divide(
@@ -438,6 +439,12 @@ class VocabParallelEmbedding(PluggableLayer):
# If parameter does not have output dim, then it should
# be copied onto all gpus (e.g. g_idx for act_order gptq).
if output_dim is None:
if (
loaded_weight.ndim == 0
and param.data.ndim == 1
and param.data.numel() == 1
):
loaded_weight = loaded_weight.reshape(1)
assert param.data.shape == loaded_weight.shape
param.data.copy_(loaded_weight)
return