[ROCm][Kimi-K3] Enable a4w4 flydsl kernels for KimiK3 (#53940)

Signed-off-by: ppalanga <ppalanga@amd.com>
Signed-off-by: Poovaiah Palangappa <poovaiah.palangappa@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Poovaiah Palangappa <poovaiah.palangappa@amd.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Hongxia Yang <62075498+hongxiayang@users.noreply.github.com>
This commit is contained in:
ppalanga
2026-09-16 01:38:02 +00:00
committed by GitHub
co-authored by Cursor Poovaiah Palangappa mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Hongxia Yang
parent c8d1cf077a
commit a4d2d9d95e
5 changed files with 127 additions and 30 deletions
@@ -385,14 +385,10 @@ def rocm_aiter_fused_experts(
gate_mode = ""
if activation == MoEActivation.SITU:
# a8w4 (VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4=1) uses the gate/up-
# interleaved (_gui_) fp8 flydsl kernels; default a16w4 SiTU stays
# separated.
gate_mode = (
GateMode.INTERLEAVE.value
if rocm_aiter_ops.is_fused_moe_situv2_a8w4_enabled()
else GateMode.SEPARATED.value
)
# SiTUv2 flydsl (VLLM_ROCM_USE_AITER_MOE_SITUV2=1) uses a4w4
# fp4 activations with separated gate/up weights (AITER #4463);
# default a16w4 SiTU also stays separated.
gate_mode = GateMode.SEPARATED.value
elif quant_config.use_mxfp4_w4a16:
gate_mode = GateMode.INTERLEAVE.value
elif activation_interleave is not None:
@@ -1572,12 +1572,6 @@ def convert_weight_to_mxfp4_moe_kernel_format(
if w2_bias is not None:
w2_bias = w2_bias.data.to(torch.float32)
import os
# TODO: Remove this once AITER is fixed
# Necessary for AITER side from crashing
os.environ["AITER_BF16_FP8_MOE_BOUND"] = "0"
if activation == MoEActivation.SITU:
from aiter.utility.fp4_utils import e8m0_shuffle
@@ -1585,9 +1579,8 @@ def convert_weight_to_mxfp4_moe_kernel_format(
fp4_dtype = torch.float4_e2m1fn_x2
e8m0_dtype = torch.float8_e8m0fnu
# a8w4 uses gate/up-interleaved flydsl kernels;
# default a16w4 keeps the separated layout.
guinterleave = rocm_aiter_ops.is_fused_moe_situv2_a8w4_enabled()
# SiTUv2 flydsl uses separated gate/up layout (a4w4).
guinterleave = False
w13 = rocm_aiter_ops.shuffle_weight_a16w4(
w13_weight.data.view(fp4_dtype), 16, guinterleave
)
@@ -1606,6 +1599,14 @@ def convert_weight_to_mxfp4_moe_kernel_format(
w2.is_shuffled = True
return (w13, w2, w13_scale, w2_scale, w13_bias, w2_bias)
import os
# Interleaved a16w4 only (DeepSeekV4 etc.). AITER uses this bound to
# pick bf16 vs fp8 activations when gate_mode is INTERLEAVE. SiTUv2
# a4w4 is separated and selects q_dtype_a independently, so the bound
# is unused on that path.
os.environ["AITER_BF16_FP8_MOE_BOUND"] = "0"
from aiter.ops.shuffle import shuffle_scale as _shuf_s
from aiter.ops.shuffle import shuffle_weight as _shuf_w