[ROCm][DI] Enable MoRI FP4 dispatch for mxfp4 w4a4 moe (#59596)
Signed-off-by: lcskrishna <lollachaitanya@gmail.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Cursor
parent
10d2c6083d
commit
833665cded
@@ -329,14 +329,24 @@ def maybe_make_prepare_finalize(
|
||||
elif moe.use_mori_kernels:
|
||||
assert quant_config is not None
|
||||
|
||||
# MXFP4 activations are dispatched as packed FP4 with their e8m0
|
||||
# block scales, which the experts consume without re-quantizing.
|
||||
use_fp4_dispatch = quant_config.use_mxfp4_w4a4
|
||||
# Note: We may want to use FP8 dispatch just to reduce
|
||||
# data movement.
|
||||
use_fp8_dispatch = (
|
||||
use_fp8_dispatch = not use_fp4_dispatch and (
|
||||
quant_config.is_per_act_token
|
||||
or quant_config.is_block_quantized
|
||||
or quant_config.is_per_tensor
|
||||
)
|
||||
if use_fp8_dispatch:
|
||||
if use_fp4_dispatch:
|
||||
# One e8m0 scale per 32-element block. token_hidden_size stays
|
||||
# the unpacked hidden size: mori sizes its token buffers by the
|
||||
# BF16 input type, which also covers the BF16 combine.
|
||||
quant_dtype = torch.float4_e2m1fn_x2
|
||||
scale_dim = moe.hidden_dim // 32
|
||||
scale_type_size = torch.float8_e8m0fnu.itemsize
|
||||
elif use_fp8_dispatch:
|
||||
# For PTPC (per token per channel) or per-tensor quant,
|
||||
# scale dim is 1. For 1x128 quant, scale dim is
|
||||
# hidden_dim // 128
|
||||
@@ -346,18 +356,20 @@ def maybe_make_prepare_finalize(
|
||||
if (quant_config.is_per_act_token or quant_config.is_per_tensor)
|
||||
else moe.hidden_dim // 128
|
||||
)
|
||||
scale_type_size = torch.float32.itemsize
|
||||
else:
|
||||
# Unquantized dispatch (e.g. AITER with defer_input_quant):
|
||||
# dispatch raw BF16/FP16 data, no scales needed.
|
||||
quant_dtype = moe.in_dtype
|
||||
scale_dim = 0
|
||||
scale_type_size = 0
|
||||
all_to_all_args = dict(
|
||||
rank=all2all_manager.rank,
|
||||
num_ep_ranks=all2all_manager.world_size,
|
||||
quant_dtype=quant_dtype,
|
||||
token_hidden_size=moe.hidden_dim,
|
||||
scale_dim=scale_dim,
|
||||
scale_type_size=0 if scale_dim == 0 else torch.float32.itemsize,
|
||||
scale_type_size=scale_type_size,
|
||||
max_num_tokens_per_dp_rank=moe.max_num_tokens,
|
||||
input_dtype=moe.in_dtype,
|
||||
num_local_experts=moe.num_experts // all2all_manager.world_size,
|
||||
@@ -370,6 +382,7 @@ def maybe_make_prepare_finalize(
|
||||
max_tokens_per_rank=moe.max_num_tokens,
|
||||
num_dispatchers=all2all_manager.world_size,
|
||||
use_fp8_dispatch=use_fp8_dispatch,
|
||||
use_fp4_dispatch=use_fp4_dispatch,
|
||||
)
|
||||
|
||||
elif moe.use_fi_nvl_two_sided_kernels:
|
||||
|
||||
@@ -352,7 +352,11 @@ def rocm_aiter_fused_experts(
|
||||
intermediate_pad = 0
|
||||
assert moe_config.hidden_dim_unpadded is not None
|
||||
assert moe_config.intermediate_size_per_partition_unpadded is not None
|
||||
hidden_pad = hidden_states.shape[1] - moe_config.hidden_dim_unpadded
|
||||
hidden_size = hidden_states.shape[1]
|
||||
if hidden_states.dtype == torch.float4_e2m1fn_x2:
|
||||
# Pre-quantized MXFP4 from MoRI dispatch, packed two per byte.
|
||||
hidden_size *= 2
|
||||
hidden_pad = hidden_size - moe_config.hidden_dim_unpadded
|
||||
intermediate_pad = (
|
||||
(
|
||||
moe_config.intermediate_size_per_partition
|
||||
@@ -448,7 +452,7 @@ class AiterExperts(mk.FusedMoEExpertsModular):
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
# When paired with MoRI, the prepare/finalize handles FP8
|
||||
# When paired with MoRI, the prepare/finalize handles FP8 or MXFP4
|
||||
# quantization during dispatch to reduce network traffic,
|
||||
# so we should not defer input quantization.
|
||||
# Otherwise, AITER fused MoE kernels handle input quantization
|
||||
@@ -516,7 +520,12 @@ class AiterExperts(mk.FusedMoEExpertsModular):
|
||||
# Workspaces are managed internally by AITER.
|
||||
workspace1 = (0,)
|
||||
workspace2 = (0,)
|
||||
output = (M, K)
|
||||
# MoRI dispatches MXFP4 activations packed two per byte, so K is
|
||||
# half the hidden size there.
|
||||
packed_fp4_input = (
|
||||
self.moe_config.use_mori_kernels and self.quant_config.use_mxfp4_w4a4
|
||||
)
|
||||
output = (M, K * 2 if packed_fp4_input else K)
|
||||
return (workspace1, workspace2, output)
|
||||
|
||||
def apply(
|
||||
|
||||
@@ -21,12 +21,15 @@ class MoriPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
max_tokens_per_rank: int,
|
||||
num_dispatchers: int,
|
||||
use_fp8_dispatch: bool = False,
|
||||
use_fp4_dispatch: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
assert not (use_fp8_dispatch and use_fp4_dispatch)
|
||||
self.mori_op = mori_op
|
||||
self.num_dispatchers_ = num_dispatchers
|
||||
self.max_tokens_per_rank = max_tokens_per_rank
|
||||
self.use_fp8_dispatch = use_fp8_dispatch
|
||||
self.use_fp4_dispatch = use_fp4_dispatch
|
||||
self._dispatch_topk_ids: torch.Tensor | None = None
|
||||
|
||||
@property
|
||||
@@ -73,7 +76,7 @@ class MoriPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
)
|
||||
scale = None
|
||||
# When defer_input_quant is True, the expert kernel handles
|
||||
# quantization internally, so skip FP8 dispatch quantization.
|
||||
# quantization internally, so skip dispatch quantization.
|
||||
if self.use_fp8_dispatch and not defer_input_quant:
|
||||
from aiter import QuantType, get_hip_quant
|
||||
|
||||
@@ -92,6 +95,14 @@ class MoriPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
)
|
||||
# mori expects one scale slot per token; broadcast.
|
||||
scale = scale.expand(a1.shape[0], 1).contiguous()
|
||||
elif self.use_fp4_dispatch and not defer_input_quant:
|
||||
from aiter import QuantType, get_hip_quant
|
||||
|
||||
# Packed MXFP4 [M, K // 2] with e8m0 scales [M, K // 32]. The
|
||||
# scales stay unshuffled: AITER fused_moe sorts them itself
|
||||
# when it receives pre-quantized FP4 input.
|
||||
quant_func = get_hip_quant(QuantType.per_1x32)
|
||||
a1, scale = quant_func(a1, shuffle=False)
|
||||
|
||||
# mori's combine() reduces over this rank's own [num_tokens, topk]
|
||||
# routing. The modular kernel rebinds topk_ids to the dispatched ids
|
||||
|
||||
Reference in New Issue
Block a user