[Quantization] add humming quantization kernel (#34556)
This commit is contained in:
@@ -60,6 +60,7 @@ WEIGHT_LOADER_V2_SUPPORTED = [
|
||||
"ModelOptFp8PbWoLinearMethod",
|
||||
"QuarkLinearMethod",
|
||||
"ModelOptNvFp4LinearMethod",
|
||||
"HummingLinearMethod",
|
||||
]
|
||||
|
||||
|
||||
@@ -245,6 +246,7 @@ class LinearBase(PluggableLayer):
|
||||
self,
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
bias: bool = False,
|
||||
skip_bias_add: bool = False,
|
||||
params_dtype: torch.dtype | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
@@ -258,6 +260,7 @@ class LinearBase(PluggableLayer):
|
||||
# Keep input parameters
|
||||
self.input_size = input_size
|
||||
self.output_size = output_size
|
||||
self.has_bias = bias
|
||||
self.skip_bias_add = skip_bias_add
|
||||
if params_dtype is None:
|
||||
params_dtype = torch.get_default_dtype()
|
||||
@@ -323,6 +326,7 @@ class ReplicatedLinear(LinearBase):
|
||||
super().__init__(
|
||||
input_size,
|
||||
output_size,
|
||||
bias,
|
||||
skip_bias_add,
|
||||
params_dtype,
|
||||
quant_config,
|
||||
@@ -458,6 +462,7 @@ class ColumnParallelLinear(LinearBase):
|
||||
super().__init__(
|
||||
input_size,
|
||||
output_size,
|
||||
bias,
|
||||
skip_bias_add,
|
||||
params_dtype,
|
||||
quant_config,
|
||||
@@ -483,6 +488,7 @@ class ColumnParallelLinear(LinearBase):
|
||||
else self.weight_loader
|
||||
),
|
||||
)
|
||||
|
||||
if bias:
|
||||
self.bias = Parameter(
|
||||
torch.empty(self.output_size_per_partition, dtype=params_dtype)
|
||||
@@ -817,8 +823,8 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
|
||||
# for the packing.
|
||||
packed_dim = getattr(param, "packed_dim", None)
|
||||
if packed_dim == output_dim:
|
||||
shard_size = shard_size // param.packed_factor
|
||||
shard_offset = shard_offset // param.packed_factor
|
||||
shard_size = round(shard_size // param.packed_factor)
|
||||
shard_offset = round(shard_offset // param.packed_factor)
|
||||
# Special case for Marlin.
|
||||
shard_size, shard_offset = adjust_marlin_shard(
|
||||
param, shard_size, shard_offset
|
||||
@@ -1252,8 +1258,8 @@ class QKVParallelLinear(ColumnParallelLinear):
|
||||
)
|
||||
|
||||
if packed_dim == output_dim:
|
||||
shard_size = shard_size // param.packed_factor
|
||||
shard_offset = shard_offset // param.packed_factor
|
||||
shard_size = round(shard_size // param.packed_factor)
|
||||
shard_offset = round(shard_offset // param.packed_factor)
|
||||
|
||||
# Special case for Marlin.
|
||||
shard_size, shard_offset = adjust_marlin_shard(
|
||||
@@ -1315,8 +1321,8 @@ class QKVParallelLinear(ColumnParallelLinear):
|
||||
# for the packing.
|
||||
packed_dim = getattr(param, "packed_dim", None)
|
||||
if packed_dim == output_dim:
|
||||
shard_size = shard_size // param.packed_factor
|
||||
shard_offset = shard_offset // param.packed_factor
|
||||
shard_size = round(shard_size // param.packed_factor)
|
||||
shard_offset = round(shard_offset // param.packed_factor)
|
||||
|
||||
# Special case for Marlin.
|
||||
shard_size, shard_offset = adjust_marlin_shard(
|
||||
@@ -1440,6 +1446,7 @@ class RowParallelLinear(LinearBase):
|
||||
super().__init__(
|
||||
input_size,
|
||||
output_size,
|
||||
bias,
|
||||
skip_bias_add,
|
||||
params_dtype,
|
||||
quant_config,
|
||||
|
||||
Reference in New Issue
Block a user