[Performance] Fused blockwise quant RMS norm (#27883)
Signed-off-by: ElizaWszola <ewszola@redhat.com> Signed-off-by: yewentao256 <zhyanwentao@126.com> Co-authored-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
co-authored by
yewentao256
parent
0044c4038c
commit
af0444bf40
@@ -436,6 +436,46 @@ def rms_norm_dynamic_per_token_quant(
|
||||
return output, scales
|
||||
|
||||
|
||||
# fused quant layer norm ops blocked
|
||||
def rms_norm_per_block_quant(
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
epsilon: float,
|
||||
quant_dtype: torch.dtype,
|
||||
group_size: list[int],
|
||||
scale_ub: torch.Tensor | None = None,
|
||||
residual: torch.Tensor | None = None,
|
||||
is_scale_transposed: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert len(group_size) == 2
|
||||
output = torch.empty_like(input, dtype=quant_dtype)
|
||||
if is_scale_transposed:
|
||||
scales = torch.empty(
|
||||
(input.shape[-1] // group_size[1], input.numel() // input.shape[-1]),
|
||||
device=input.device,
|
||||
dtype=torch.float32,
|
||||
).transpose(0, 1)
|
||||
else:
|
||||
scales = torch.empty(
|
||||
(input.numel() // input.shape[-1], input.shape[-1] // group_size[1]),
|
||||
device=input.device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
|
||||
torch.ops._C.rms_norm_per_block_quant(
|
||||
output,
|
||||
input,
|
||||
weight,
|
||||
scales,
|
||||
epsilon,
|
||||
scale_ub,
|
||||
residual,
|
||||
group_size[1],
|
||||
is_scale_transposed,
|
||||
)
|
||||
return output, scales
|
||||
|
||||
|
||||
# quantization ops
|
||||
# awq
|
||||
def awq_dequantize(
|
||||
|
||||
Reference in New Issue
Block a user