[Bugfix] Fix quant RMS norm fusion for quantization with TMA-aligned scales (#33255)
Signed-off-by: ElizaWszola <ewszola@redhat.com> Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
This commit is contained in:
co-authored by
Luka Govedič
parent
a49ea5a58f
commit
a88b3be7c4
+24
-5
@@ -450,15 +450,30 @@ def rms_norm_per_block_quant(
|
||||
scale_ub: torch.Tensor | None = None,
|
||||
residual: torch.Tensor | None = None,
|
||||
is_scale_transposed: bool = False,
|
||||
tma_alignment: int = 0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert len(group_size) == 2
|
||||
output = torch.empty_like(input, dtype=quant_dtype)
|
||||
if is_scale_transposed:
|
||||
scales = torch.empty(
|
||||
(input.shape[-1] // group_size[1], input.numel() // input.shape[-1]),
|
||||
device=input.device,
|
||||
dtype=torch.float32,
|
||||
).transpose(0, 1)
|
||||
if tma_alignment == 0:
|
||||
scales = torch.empty(
|
||||
(input.shape[-1] // group_size[1], input.numel() // input.shape[-1]),
|
||||
device=input.device,
|
||||
dtype=torch.float32,
|
||||
).transpose(0, 1)
|
||||
else:
|
||||
m = input.shape[-2]
|
||||
sf_k = input.shape[-1] // group_size[1]
|
||||
tma_aligned_m = (m + tma_alignment - 1) // tma_alignment * tma_alignment
|
||||
shape = input.shape[:-2] + (m, sf_k)
|
||||
stride = (
|
||||
(1, tma_aligned_m)
|
||||
if input.dim() == 2
|
||||
else (tma_aligned_m * sf_k, 1, tma_aligned_m)
|
||||
)
|
||||
scales = torch.empty_strided(
|
||||
shape, stride, device=input.device, dtype=torch.float32
|
||||
)
|
||||
else:
|
||||
scales = torch.empty(
|
||||
(input.numel() // input.shape[-1], input.shape[-1] // group_size[1]),
|
||||
@@ -466,6 +481,10 @@ def rms_norm_per_block_quant(
|
||||
dtype=torch.float32,
|
||||
)
|
||||
|
||||
assert tma_alignment in [0, 4], "Expected TMA alignment 0 or 4, but got " + str(
|
||||
tma_alignment
|
||||
)
|
||||
|
||||
torch.ops._C.rms_norm_per_block_quant(
|
||||
output,
|
||||
input,
|
||||
|
||||
Reference in New Issue
Block a user