[ROCm][CI] Enable AITER FP8/unquantized cases in modular-kernel sweep + fix MoRI per-tensor FP8 dispatch (#57074)
Signed-off-by: Divakar Verma <divakar.verma@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Andreas Karatzas
parent
a615f53364
commit
95f4925c3a
@@ -284,13 +284,20 @@ def maybe_make_prepare_finalize(
|
||||
# Note: We may want to use FP8 dispatch just to reduce
|
||||
# data movement.
|
||||
use_fp8_dispatch = (
|
||||
quant_config.is_per_act_token or quant_config.is_block_quantized
|
||||
quant_config.is_per_act_token
|
||||
or quant_config.is_block_quantized
|
||||
or quant_config.is_per_tensor
|
||||
)
|
||||
if use_fp8_dispatch:
|
||||
# For PTPC (per token per channel) quant, scale dim is 1
|
||||
# For 1x128 quant, scale dim is hidden_dim // 128
|
||||
# For PTPC (per token per channel) or per-tensor quant,
|
||||
# scale dim is 1. For 1x128 quant, scale dim is
|
||||
# hidden_dim // 128
|
||||
quant_dtype = quant_config.quant_dtype
|
||||
scale_dim = 1 if quant_config.is_per_act_token else moe.hidden_dim // 128
|
||||
scale_dim = (
|
||||
1
|
||||
if (quant_config.is_per_act_token or quant_config.is_per_tensor)
|
||||
else moe.hidden_dim // 128
|
||||
)
|
||||
else:
|
||||
# Unquantized dispatch (e.g. AITER with defer_input_quant):
|
||||
# dispatch raw BF16/FP16 data, no scales needed.
|
||||
|
||||
@@ -82,6 +82,15 @@ class MoriPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
elif quant_config.is_per_act_token:
|
||||
quant_func = get_hip_quant(QuantType.per_Token)
|
||||
a1, scale = quant_func(a1, quant_dtype=current_platform.fp8_dtype())
|
||||
elif quant_config.is_per_tensor:
|
||||
quant_func = get_hip_quant(QuantType.per_Tensor)
|
||||
a1, scale = quant_func(
|
||||
a1,
|
||||
scale=quant_config.a1_scale,
|
||||
quant_dtype=current_platform.fp8_dtype(),
|
||||
)
|
||||
# mori expects one scale slot per token; broadcast.
|
||||
scale = scale.expand(a1.shape[0], 1).contiguous()
|
||||
|
||||
(
|
||||
dispatch_a1,
|
||||
|
||||
Reference in New Issue
Block a user