[Bugfix][Quantization] Give LM heads standard linear metadata (#58444)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
-2
@@ -119,8 +119,6 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
|
||||
layer.output_size_per_partition = output_size_per_partition
|
||||
layer.output_partition_sizes = output_partition_sizes
|
||||
layer.params_dtype = params_dtype
|
||||
if not hasattr(layer, "has_bias"):
|
||||
layer.has_bias = False
|
||||
|
||||
mp_linear_kernel_config = MPLinearLayerConfig(
|
||||
full_weight_shape=(input_size, output_size),
|
||||
|
||||
-2
@@ -147,8 +147,6 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
|
||||
layer.output_size_per_partition = output_size_per_partition
|
||||
layer.output_partition_sizes = output_partition_sizes
|
||||
layer.params_dtype = params_dtype
|
||||
if not hasattr(layer, "has_bias"):
|
||||
layer.has_bias = False
|
||||
|
||||
mp_config = MPLinearLayerConfig(
|
||||
full_weight_shape=(input_size, output_size),
|
||||
|
||||
-2
@@ -147,8 +147,6 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
|
||||
layer.output_size_per_partition = output_size_per_partition
|
||||
layer.output_partition_sizes = output_partition_sizes
|
||||
layer.params_dtype = params_dtype
|
||||
if not hasattr(layer, "has_bias"):
|
||||
layer.has_bias = False
|
||||
|
||||
mp_config = MPLinearLayerConfig(
|
||||
full_weight_shape=(input_size, output_size),
|
||||
|
||||
-4
@@ -97,12 +97,8 @@ class CompressedTensorsWNA8O8Int(CompressedTensorsScheme):
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
layer.input_size_per_partition = input_size_per_partition
|
||||
layer.output_size_per_partition = output_size_per_partition
|
||||
# Set for kernels' weight prep; also covers ParallelLMHead, which does
|
||||
# not set these in __init__.
|
||||
layer.output_partition_sizes = output_partition_sizes
|
||||
layer.params_dtype = params_dtype
|
||||
if not hasattr(layer, "has_bias"):
|
||||
layer.has_bias = False
|
||||
|
||||
mp_config = MPLinearLayerConfig(
|
||||
full_weight_shape=(input_size, output_size),
|
||||
|
||||
@@ -2508,12 +2508,6 @@ class ModelOptLinearMethod(LinearMethodBase):
|
||||
layer.logical_widths = output_partition_sizes
|
||||
layer.input_size_per_partition = input_size_per_partition
|
||||
layer.output_size_per_partition = sum(output_partition_sizes)
|
||||
# Humming reads both off the layer in
|
||||
# prepare_humming_linear_layer_config. LinearBase sets them itself;
|
||||
# ParallelLMHead does not, so supply them here.
|
||||
layer.output_partition_sizes = output_partition_sizes
|
||||
if not hasattr(layer, "has_bias"):
|
||||
layer.has_bias = getattr(layer, "bias", None) is not None
|
||||
shapes = Shapes(output_partition_sizes, input_size_per_partition, params_dtype)
|
||||
|
||||
self.wkey.create_weights(layer, WEIGHT, self.ctx, shapes, weight_loader)
|
||||
|
||||
@@ -278,12 +278,12 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
self.num_embeddings = num_embeddings
|
||||
self.padding_size = padding_size
|
||||
self.org_vocab_size = org_num_embeddings or num_embeddings
|
||||
num_added_embeddings = num_embeddings - self.org_vocab_size
|
||||
self.num_added_embeddings = num_embeddings - self.org_vocab_size
|
||||
self.org_vocab_size_padded = pad_vocab_size(
|
||||
self.org_vocab_size, self.padding_size
|
||||
)
|
||||
self.num_embeddings_padded = pad_vocab_size(
|
||||
self.org_vocab_size_padded + num_added_embeddings, self.padding_size
|
||||
self.org_vocab_size_padded + self.num_added_embeddings, self.padding_size
|
||||
)
|
||||
assert self.org_vocab_size_padded <= self.num_embeddings_padded
|
||||
|
||||
@@ -296,6 +296,18 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
self.tp_size,
|
||||
)
|
||||
self.embedding_dim = embedding_dim
|
||||
# Divide the weight matrix along the vocabulary dimension.
|
||||
self.num_embeddings_per_partition = divide(
|
||||
self.num_embeddings_padded, self.tp_size
|
||||
)
|
||||
|
||||
# Quantization methods share the same weight factory as linear layers,
|
||||
# so setup standard linear metadata.
|
||||
self.input_size = self.input_size_per_partition = embedding_dim
|
||||
self.output_size = self.num_embeddings_padded
|
||||
self.output_size_per_partition = self.num_embeddings_per_partition
|
||||
self.output_partition_sizes = [self.output_size_per_partition]
|
||||
self.prefix = prefix
|
||||
|
||||
# Avoid overriding a preselected model-specific method with generic
|
||||
# config-based dispatch.
|
||||
@@ -322,11 +334,6 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
if params_dtype is None:
|
||||
params_dtype = torch.get_default_dtype()
|
||||
self.params_dtype = params_dtype
|
||||
# Divide the weight matrix along the vocabulary dimension.
|
||||
self.num_added_embeddings = self.num_embeddings - self.org_vocab_size
|
||||
self.num_embeddings_per_partition = divide(
|
||||
self.num_embeddings_padded, self.tp_size
|
||||
)
|
||||
assert (
|
||||
self.shard_indices.num_elements_padded == self.num_embeddings_per_partition
|
||||
)
|
||||
@@ -349,10 +356,10 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
|
||||
self.quant_method.create_weights(
|
||||
self,
|
||||
self.embedding_dim,
|
||||
[self.num_embeddings_per_partition],
|
||||
self.embedding_dim,
|
||||
self.num_embeddings_padded,
|
||||
self.input_size_per_partition,
|
||||
self.output_partition_sizes,
|
||||
self.input_size,
|
||||
self.output_size,
|
||||
params_dtype=params_dtype,
|
||||
weight_loader=self.weight_loader,
|
||||
)
|
||||
@@ -600,6 +607,7 @@ class ParallelLMHead(VocabParallelEmbedding):
|
||||
*,
|
||||
disable_tp: bool = False,
|
||||
):
|
||||
self.has_bias = bias
|
||||
super().__init__(
|
||||
num_embeddings,
|
||||
embedding_dim,
|
||||
|
||||
Reference in New Issue
Block a user