[Quantization] add humming quantization kernel (#34556)

This commit is contained in:
Jinzhen Lin
2026-04-24 09:29:44 -04:00
committed by GitHub
parent c9d3c6e6af
commit 9f771b3ab9
10 changed files with 1948 additions and 9 deletions
+13 -6
View File
@@ -60,6 +60,7 @@ WEIGHT_LOADER_V2_SUPPORTED = [
"ModelOptFp8PbWoLinearMethod",
"QuarkLinearMethod",
"ModelOptNvFp4LinearMethod",
"HummingLinearMethod",
]
@@ -245,6 +246,7 @@ class LinearBase(PluggableLayer):
self,
input_size: int,
output_size: int,
bias: bool = False,
skip_bias_add: bool = False,
params_dtype: torch.dtype | None = None,
quant_config: QuantizationConfig | None = None,
@@ -258,6 +260,7 @@ class LinearBase(PluggableLayer):
# Keep input parameters
self.input_size = input_size
self.output_size = output_size
self.has_bias = bias
self.skip_bias_add = skip_bias_add
if params_dtype is None:
params_dtype = torch.get_default_dtype()
@@ -323,6 +326,7 @@ class ReplicatedLinear(LinearBase):
super().__init__(
input_size,
output_size,
bias,
skip_bias_add,
params_dtype,
quant_config,
@@ -458,6 +462,7 @@ class ColumnParallelLinear(LinearBase):
super().__init__(
input_size,
output_size,
bias,
skip_bias_add,
params_dtype,
quant_config,
@@ -483,6 +488,7 @@ class ColumnParallelLinear(LinearBase):
else self.weight_loader
),
)
if bias:
self.bias = Parameter(
torch.empty(self.output_size_per_partition, dtype=params_dtype)
@@ -817,8 +823,8 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
# for the packing.
packed_dim = getattr(param, "packed_dim", None)
if packed_dim == output_dim:
shard_size = shard_size // param.packed_factor
shard_offset = shard_offset // param.packed_factor
shard_size = round(shard_size // param.packed_factor)
shard_offset = round(shard_offset // param.packed_factor)
# Special case for Marlin.
shard_size, shard_offset = adjust_marlin_shard(
param, shard_size, shard_offset
@@ -1252,8 +1258,8 @@ class QKVParallelLinear(ColumnParallelLinear):
)
if packed_dim == output_dim:
shard_size = shard_size // param.packed_factor
shard_offset = shard_offset // param.packed_factor
shard_size = round(shard_size // param.packed_factor)
shard_offset = round(shard_offset // param.packed_factor)
# Special case for Marlin.
shard_size, shard_offset = adjust_marlin_shard(
@@ -1315,8 +1321,8 @@ class QKVParallelLinear(ColumnParallelLinear):
# for the packing.
packed_dim = getattr(param, "packed_dim", None)
if packed_dim == output_dim:
shard_size = shard_size // param.packed_factor
shard_offset = shard_offset // param.packed_factor
shard_size = round(shard_size // param.packed_factor)
shard_offset = round(shard_offset // param.packed_factor)
# Special case for Marlin.
shard_size, shard_offset = adjust_marlin_shard(
@@ -1440,6 +1446,7 @@ class RowParallelLinear(LinearBase):
super().__init__(
input_size,
output_size,
bias,
skip_bias_add,
params_dtype,
quant_config,