[Perf][MoE] Skip the TRTLLM fp8-block activation-scale transpose copy (#61773)
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
@@ -249,6 +249,8 @@ class FusedMoEQuantConfig:
|
||||
_w1: FusedMoEQuantDesc
|
||||
_w2: FusedMoEQuantDesc
|
||||
is_scale_swizzled: bool = True
|
||||
# Emit block-quant activation scales column-major (physical [G, M] layout).
|
||||
is_scale_transposed: bool = False
|
||||
|
||||
# MXFP4-specific TRTLLM parameters for SwiGLU activation clamping.
|
||||
# These correspond to gemm1_alpha, gemm1_beta, gemm1_clamp_limit
|
||||
@@ -503,6 +505,7 @@ class FusedMoEQuantConfig:
|
||||
w2_zp: torch.Tensor | None = None,
|
||||
weight_dtype: torch.dtype | str | None = None,
|
||||
is_scale_swizzled: bool = True,
|
||||
is_scale_transposed: bool = False,
|
||||
gemm1_alpha: float | None = None,
|
||||
gemm1_beta: float | None = None,
|
||||
gemm1_clamp_limit: float | None = None,
|
||||
@@ -537,6 +540,9 @@ class FusedMoEQuantConfig:
|
||||
- is_scale_swizzled: Whether the activation scale-factor layout is
|
||||
swizzled. Pass through to the underlying quantization kernel for
|
||||
dtypes that distinguish layouts (nvfp4, mxfp8). Defaults to True.
|
||||
- is_scale_transposed: Whether block-quantized activation scales are
|
||||
emitted column-major (physical [num_groups, num_tokens] layout) by
|
||||
the quantize step. Defaults to False.
|
||||
- gemm1_alpha: Optional MXFP4 TRTLLM SwiGLU alpha parameter.
|
||||
- gemm1_beta: Optional MXFP4 TRTLLM SwiGLU beta parameter.
|
||||
- gemm1_clamp_limit: Optional MXFP4 TRTLLM SwiGLU clamp limit.
|
||||
@@ -573,6 +579,7 @@ class FusedMoEQuantConfig:
|
||||
weight_dtype, w_shape, w2_scale, g2_alphas, w2_zp, w2_bias
|
||||
),
|
||||
is_scale_swizzled=is_scale_swizzled,
|
||||
is_scale_transposed=is_scale_transposed,
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
gemm1_clamp_limit=gemm1_clamp_limit,
|
||||
@@ -597,6 +604,7 @@ def fp8_w8a8_moe_quant_config(
|
||||
a2_gscale: torch.Tensor | None = None,
|
||||
g1_alphas: torch.Tensor | None = None,
|
||||
g2_alphas: torch.Tensor | None = None,
|
||||
is_scale_transposed: bool = False,
|
||||
gemm1_alpha: float | None = None,
|
||||
gemm1_beta: float | None = None,
|
||||
gemm1_clamp_limit: float | None = None,
|
||||
@@ -617,6 +625,7 @@ def fp8_w8a8_moe_quant_config(
|
||||
per_act_token_quant=per_act_token_quant,
|
||||
per_out_ch_quant=per_out_ch_quant,
|
||||
block_shape=block_shape,
|
||||
is_scale_transposed=is_scale_transposed,
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
gemm1_clamp_limit=gemm1_clamp_limit,
|
||||
|
||||
@@ -841,6 +841,11 @@ def make_fp8_moe_quant_config(
|
||||
block_shape=block_shape,
|
||||
per_act_token_quant=per_act_token_quant,
|
||||
per_out_ch_quant=per_out_ch_quant,
|
||||
# TRTLLM fp8-block consumes activation scales as [K/128, M]; emit them
|
||||
# transposed from the quant kernel instead of copying afterwards.
|
||||
is_scale_transposed=(
|
||||
fp8_backend == Fp8MoeBackend.FLASHINFER_TRTLLM and block_shape == [128, 128]
|
||||
),
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
gemm1_clamp_limit=swiglu_limit,
|
||||
|
||||
@@ -31,6 +31,7 @@ def _quantize_input(
|
||||
per_act_token_quant=quant_config.per_act_token_quant,
|
||||
block_shape=quant_config.block_shape,
|
||||
is_scale_swizzled=quant_config.is_scale_swizzled,
|
||||
is_scale_transposed=quant_config.is_scale_transposed,
|
||||
mx_alignment=quant_config.mx_alignment,
|
||||
)
|
||||
|
||||
|
||||
@@ -325,6 +325,7 @@ def _fp8_quantize(
|
||||
A_scale: torch.Tensor | None,
|
||||
per_act_token: bool,
|
||||
block_shape: list[int] | None = None,
|
||||
is_scale_transposed: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Perform fp8 quantization on the inputs. If a block_shape
|
||||
is provided, the output will be blocked.
|
||||
@@ -339,7 +340,9 @@ def _fp8_quantize(
|
||||
assert not per_act_token
|
||||
assert len(block_shape) == 2
|
||||
_, block_k = block_shape[0], block_shape[1]
|
||||
A, A_scale = per_token_group_quant_fp8(A, block_k)
|
||||
A, A_scale = per_token_group_quant_fp8(
|
||||
A, block_k, column_major_scales=is_scale_transposed
|
||||
)
|
||||
assert cdiv(A.size(-1), block_k) == A_scale.size(-1)
|
||||
|
||||
return A, A_scale
|
||||
@@ -462,6 +465,7 @@ def moe_kernel_quantize_input(
|
||||
per_act_token_quant: bool,
|
||||
block_shape: list[int] | None = None,
|
||||
is_scale_swizzled: bool = True,
|
||||
is_scale_transposed: bool = False,
|
||||
quantization_emulation: bool = False,
|
||||
mx_alignment: int = 0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
@@ -469,7 +473,9 @@ def moe_kernel_quantize_input(
|
||||
if quantization_emulation:
|
||||
return _fp8_quantize_dequantize(A, A_scale)
|
||||
else:
|
||||
return _fp8_quantize(A, A_scale, per_act_token_quant, block_shape)
|
||||
return _fp8_quantize(
|
||||
A, A_scale, per_act_token_quant, block_shape, is_scale_transposed
|
||||
)
|
||||
elif quant_dtype == torch.int8:
|
||||
if quantization_emulation:
|
||||
raise NotImplementedError(
|
||||
|
||||
Reference in New Issue
Block a user