[ROCm][gpt-oss] Pass GateMode.INTERLEAVE for MXFP4 W4A16 fused MoE (#44893)

Signed-off-by: Rohan Potdar <rohan.potdar@amd.com>
Signed-off-by: Rohan138 <rohanpotdar138@gmail.com>
Signed-off-by: Rohan Potdar <66227218+Rohan138@users.noreply.github.com>
This commit is contained in:
Rohan Potdar
2026-06-12 01:02:04 -05:00
committed by GitHub
parent 39dee1114a
commit fe04238292
2 changed files with 40 additions and 0 deletions
+24
View File
@@ -167,6 +167,7 @@ def _rocm_aiter_fused_moe_impl(
output_dtype: torch.dtype | None = None,
hidden_pad: int = 0,
intermediate_pad: int = 0,
gate_mode: str = "",
bias1: torch.Tensor | None = None,
bias2: torch.Tensor | None = None,
moe_sorting_dispatch_policy: int = 0,
@@ -177,6 +178,10 @@ def _rocm_aiter_fused_moe_impl(
activation = ActivationType(activation_method)
quant_type = QuantType(quant_method)
extra_kwargs: dict = {}
if gate_mode and rocm_aiter_ops.fused_moe_supports_gate_mode():
extra_kwargs["gate_mode"] = gate_mode
return fused_moe(
hidden_states,
w1,
@@ -198,6 +203,7 @@ def _rocm_aiter_fused_moe_impl(
bias1=bias1,
bias2=bias2,
moe_sorting_dispatch_policy=moe_sorting_dispatch_policy,
**extra_kwargs,
)
@@ -219,6 +225,7 @@ def _rocm_aiter_fused_moe_fake(
output_dtype: torch.dtype | None = None,
hidden_pad: int = 0,
intermediate_pad: int = 0,
gate_mode: str = "",
bias1: torch.Tensor | None = None,
bias2: torch.Tensor | None = None,
moe_sorting_dispatch_policy: int = 0,
@@ -1804,6 +1811,21 @@ class rocm_aiter_ops:
except (ImportError, ModuleNotFoundError):
return False
@classmethod
@if_aiter_supported
@functools.cache
def fused_moe_supports_gate_mode(cls) -> bool:
"""Probe whether the installed aiter.fused_moe accepts `gate_mode`.
Added in https://github.com/ROCm/aiter/pull/3123 (>=0.1.14).
Builds with older AITER must omit this argument.
"""
import inspect
from aiter.fused_moe import fused_moe
return "gate_mode" in inspect.signature(fused_moe).parameters
@staticmethod
@if_aiter_supported
def register_ops_once() -> None:
@@ -2172,6 +2194,7 @@ class rocm_aiter_ops:
output_dtype: torch.dtype | None = None,
hidden_pad: int = 0,
intermediate_pad: int = 0,
gate_mode: str = "",
bias1: torch.Tensor | None = None,
bias2: torch.Tensor | None = None,
moe_sorting_dispatch_policy: int = 0,
@@ -2194,6 +2217,7 @@ class rocm_aiter_ops:
output_dtype,
hidden_pad,
intermediate_pad,
gate_mode,
bias1,
bias2,
moe_sorting_dispatch_policy,
@@ -351,6 +351,21 @@ def rocm_aiter_fused_experts(
intermediate_pad // 64 * 64 * (2 if moe_config.tp_size == 1 else 1)
)
# https://github.com/ROCm/aiter/pull/3123 specialized the AITER stage1 GEMMs
# for interleaved vs separated gate and up weights.
# For gpt-oss i.e. use_mxfp4_w4a16=True, the weights are shuffled by
# `rocm_aiter_ops.shuffle_weight_a16w4` in `oracle/mxfp4.py`,
# which always sets `is_guinterleave=True`.
# Hence, we pass in GateMode.INTERLEAVE to match the weight shuffling.
gate_mode = ""
if quant_config.use_mxfp4_w4a16:
try:
from aiter.ops.flydsl.moe_common import GateMode
gate_mode = GateMode.INTERLEAVE.value
except ImportError:
pass
return rocm_aiter_ops.fused_moe(
hidden_states,
w1,
@@ -369,6 +384,7 @@ def rocm_aiter_fused_experts(
output_dtype=output_dtype,
hidden_pad=hidden_pad,
intermediate_pad=intermediate_pad,
gate_mode=gate_mode,
bias1=quant_config.w1_bias if quant_config.use_mxfp4_w4a16 else None,
bias2=quant_config.w2_bias if quant_config.use_mxfp4_w4a16 else None,
moe_sorting_dispatch_policy=moe_sorting_dispatch_policy,