Add LM head quantization support for ModelOpt (#42124)
Signed-off-by: weimingc <17592131+meenchen@users.noreply.github.com>
This commit is contained in:
@@ -290,6 +290,7 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
|
||||
if params_dtype is None:
|
||||
params_dtype = torch.get_default_dtype()
|
||||
self.params_dtype = params_dtype
|
||||
# Divide the weight matrix along the vocabulary dimension.
|
||||
self.num_added_embeddings = self.num_embeddings - self.org_vocab_size
|
||||
self.num_embeddings_per_partition = divide(
|
||||
@@ -438,6 +439,12 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
# If parameter does not have output dim, then it should
|
||||
# be copied onto all gpus (e.g. g_idx for act_order gptq).
|
||||
if output_dim is None:
|
||||
if (
|
||||
loaded_weight.ndim == 0
|
||||
and param.data.ndim == 1
|
||||
and param.data.numel() == 1
|
||||
):
|
||||
loaded_weight = loaded_weight.reshape(1)
|
||||
assert param.data.shape == loaded_weight.shape
|
||||
param.data.copy_(loaded_weight)
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user