[Perf][MoE] Skip the TRTLLM fp8-block activation-scale transpose copy (#61773)

Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
Thien Tran
2026-10-15 12:09:07 -07:00
committed by GitHub
parent 4d58fa254a
commit 82b9ccaa79
5 changed files with 77 additions and 14 deletions
@@ -325,6 +325,7 @@ def _fp8_quantize(
A_scale: torch.Tensor | None,
per_act_token: bool,
block_shape: list[int] | None = None,
is_scale_transposed: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Perform fp8 quantization on the inputs. If a block_shape
is provided, the output will be blocked.
@@ -339,7 +340,9 @@ def _fp8_quantize(
assert not per_act_token
assert len(block_shape) == 2
_, block_k = block_shape[0], block_shape[1]
A, A_scale = per_token_group_quant_fp8(A, block_k)
A, A_scale = per_token_group_quant_fp8(
A, block_k, column_major_scales=is_scale_transposed
)
assert cdiv(A.size(-1), block_k) == A_scale.size(-1)
return A, A_scale
@@ -462,6 +465,7 @@ def moe_kernel_quantize_input(
per_act_token_quant: bool,
block_shape: list[int] | None = None,
is_scale_swizzled: bool = True,
is_scale_transposed: bool = False,
quantization_emulation: bool = False,
mx_alignment: int = 0,
) -> tuple[torch.Tensor, torch.Tensor | None]:
@@ -469,7 +473,9 @@ def moe_kernel_quantize_input(
if quantization_emulation:
return _fp8_quantize_dequantize(A, A_scale)
else:
return _fp8_quantize(A, A_scale, per_act_token_quant, block_shape)
return _fp8_quantize(
A, A_scale, per_act_token_quant, block_shape, is_scale_transposed
)
elif quant_dtype == torch.int8:
if quantization_emulation:
raise NotImplementedError(