[Bugfix][Quantization] Give LM heads standard linear metadata (#58444)

Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Misha Goin
2026-09-24 16:39:49 -07:00
committed by GitHub
parent 9ac3b7c69f
commit 4f72bd0152
8 changed files with 93 additions and 70 deletions
@@ -119,8 +119,6 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
layer.output_size_per_partition = output_size_per_partition
layer.output_partition_sizes = output_partition_sizes
layer.params_dtype = params_dtype
if not hasattr(layer, "has_bias"):
layer.has_bias = False
mp_linear_kernel_config = MPLinearLayerConfig(
full_weight_shape=(input_size, output_size),
@@ -147,8 +147,6 @@ class CompressedTensorsWNA4Int(CompressedTensorsScheme):
layer.output_size_per_partition = output_size_per_partition
layer.output_partition_sizes = output_partition_sizes
layer.params_dtype = params_dtype
if not hasattr(layer, "has_bias"):
layer.has_bias = False
mp_config = MPLinearLayerConfig(
full_weight_shape=(input_size, output_size),
@@ -147,8 +147,6 @@ class CompressedTensorsWNA8Int(CompressedTensorsScheme):
layer.output_size_per_partition = output_size_per_partition
layer.output_partition_sizes = output_partition_sizes
layer.params_dtype = params_dtype
if not hasattr(layer, "has_bias"):
layer.has_bias = False
mp_config = MPLinearLayerConfig(
full_weight_shape=(input_size, output_size),
@@ -97,12 +97,8 @@ class CompressedTensorsWNA8O8Int(CompressedTensorsScheme):
output_size_per_partition = sum(output_partition_sizes)
layer.input_size_per_partition = input_size_per_partition
layer.output_size_per_partition = output_size_per_partition
# Set for kernels' weight prep; also covers ParallelLMHead, which does
# not set these in __init__.
layer.output_partition_sizes = output_partition_sizes
layer.params_dtype = params_dtype
if not hasattr(layer, "has_bias"):
layer.has_bias = False
mp_config = MPLinearLayerConfig(
full_weight_shape=(input_size, output_size),
@@ -2508,12 +2508,6 @@ class ModelOptLinearMethod(LinearMethodBase):
layer.logical_widths = output_partition_sizes
layer.input_size_per_partition = input_size_per_partition
layer.output_size_per_partition = sum(output_partition_sizes)
# Humming reads both off the layer in
# prepare_humming_linear_layer_config. LinearBase sets them itself;
# ParallelLMHead does not, so supply them here.
layer.output_partition_sizes = output_partition_sizes
if not hasattr(layer, "has_bias"):
layer.has_bias = getattr(layer, "bias", None) is not None
shapes = Shapes(output_partition_sizes, input_size_per_partition, params_dtype)
self.wkey.create_weights(layer, WEIGHT, self.ctx, shapes, weight_loader)
@@ -278,12 +278,12 @@ class VocabParallelEmbedding(PluggableLayer):
self.num_embeddings = num_embeddings
self.padding_size = padding_size
self.org_vocab_size = org_num_embeddings or num_embeddings
num_added_embeddings = num_embeddings - self.org_vocab_size
self.num_added_embeddings = num_embeddings - self.org_vocab_size
self.org_vocab_size_padded = pad_vocab_size(
self.org_vocab_size, self.padding_size
)
self.num_embeddings_padded = pad_vocab_size(
self.org_vocab_size_padded + num_added_embeddings, self.padding_size
self.org_vocab_size_padded + self.num_added_embeddings, self.padding_size
)
assert self.org_vocab_size_padded <= self.num_embeddings_padded
@@ -296,6 +296,18 @@ class VocabParallelEmbedding(PluggableLayer):
self.tp_size,
)
self.embedding_dim = embedding_dim
# Divide the weight matrix along the vocabulary dimension.
self.num_embeddings_per_partition = divide(
self.num_embeddings_padded, self.tp_size
)
# Quantization methods share the same weight factory as linear layers,
# so setup standard linear metadata.
self.input_size = self.input_size_per_partition = embedding_dim
self.output_size = self.num_embeddings_padded
self.output_size_per_partition = self.num_embeddings_per_partition
self.output_partition_sizes = [self.output_size_per_partition]
self.prefix = prefix
# Avoid overriding a preselected model-specific method with generic
# config-based dispatch.
@@ -322,11 +334,6 @@ class VocabParallelEmbedding(PluggableLayer):
if params_dtype is None:
params_dtype = torch.get_default_dtype()
self.params_dtype = params_dtype
# Divide the weight matrix along the vocabulary dimension.
self.num_added_embeddings = self.num_embeddings - self.org_vocab_size
self.num_embeddings_per_partition = divide(
self.num_embeddings_padded, self.tp_size
)
assert (
self.shard_indices.num_elements_padded == self.num_embeddings_per_partition
)
@@ -349,10 +356,10 @@ class VocabParallelEmbedding(PluggableLayer):
self.quant_method.create_weights(
self,
self.embedding_dim,
[self.num_embeddings_per_partition],
self.embedding_dim,
self.num_embeddings_padded,
self.input_size_per_partition,
self.output_partition_sizes,
self.input_size,
self.output_size,
params_dtype=params_dtype,
weight_loader=self.weight_loader,
)
@@ -600,6 +607,7 @@ class ParallelLMHead(VocabParallelEmbedding):
*,
disable_tp: bool = False,
):
self.has_bias = bias
super().__init__(
num_embeddings,
embedding_dim,