[ROCm][Kimi-K3] Enable a4w4 flydsl kernels for KimiK3 (#53940)
Signed-off-by: ppalanga <ppalanga@amd.com> Signed-off-by: Poovaiah Palangappa <poovaiah.palangappa@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Poovaiah Palangappa <poovaiah.palangappa@amd.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Hongxia Yang <62075498+hongxiayang@users.noreply.github.com>
This commit is contained in:
co-authored by
Cursor
Poovaiah Palangappa
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Hongxia Yang
parent
c8d1cf077a
commit
a4d2d9d95e
@@ -385,14 +385,10 @@ def rocm_aiter_fused_experts(
|
||||
|
||||
gate_mode = ""
|
||||
if activation == MoEActivation.SITU:
|
||||
# a8w4 (VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4=1) uses the gate/up-
|
||||
# interleaved (_gui_) fp8 flydsl kernels; default a16w4 SiTU stays
|
||||
# separated.
|
||||
gate_mode = (
|
||||
GateMode.INTERLEAVE.value
|
||||
if rocm_aiter_ops.is_fused_moe_situv2_a8w4_enabled()
|
||||
else GateMode.SEPARATED.value
|
||||
)
|
||||
# SiTUv2 flydsl (VLLM_ROCM_USE_AITER_MOE_SITUV2=1) uses a4w4
|
||||
# fp4 activations with separated gate/up weights (AITER #4463);
|
||||
# default a16w4 SiTU also stays separated.
|
||||
gate_mode = GateMode.SEPARATED.value
|
||||
elif quant_config.use_mxfp4_w4a16:
|
||||
gate_mode = GateMode.INTERLEAVE.value
|
||||
elif activation_interleave is not None:
|
||||
|
||||
@@ -1572,12 +1572,6 @@ def convert_weight_to_mxfp4_moe_kernel_format(
|
||||
if w2_bias is not None:
|
||||
w2_bias = w2_bias.data.to(torch.float32)
|
||||
|
||||
import os
|
||||
|
||||
# TODO: Remove this once AITER is fixed
|
||||
# Necessary for AITER side from crashing
|
||||
os.environ["AITER_BF16_FP8_MOE_BOUND"] = "0"
|
||||
|
||||
if activation == MoEActivation.SITU:
|
||||
from aiter.utility.fp4_utils import e8m0_shuffle
|
||||
|
||||
@@ -1585,9 +1579,8 @@ def convert_weight_to_mxfp4_moe_kernel_format(
|
||||
|
||||
fp4_dtype = torch.float4_e2m1fn_x2
|
||||
e8m0_dtype = torch.float8_e8m0fnu
|
||||
# a8w4 uses gate/up-interleaved flydsl kernels;
|
||||
# default a16w4 keeps the separated layout.
|
||||
guinterleave = rocm_aiter_ops.is_fused_moe_situv2_a8w4_enabled()
|
||||
# SiTUv2 flydsl uses separated gate/up layout (a4w4).
|
||||
guinterleave = False
|
||||
w13 = rocm_aiter_ops.shuffle_weight_a16w4(
|
||||
w13_weight.data.view(fp4_dtype), 16, guinterleave
|
||||
)
|
||||
@@ -1606,6 +1599,14 @@ def convert_weight_to_mxfp4_moe_kernel_format(
|
||||
w2.is_shuffled = True
|
||||
return (w13, w2, w13_scale, w2_scale, w13_bias, w2_bias)
|
||||
|
||||
import os
|
||||
|
||||
# Interleaved a16w4 only (DeepSeekV4 etc.). AITER uses this bound to
|
||||
# pick bf16 vs fp8 activations when gate_mode is INTERLEAVE. SiTUv2
|
||||
# a4w4 is separated and selects q_dtype_a independently, so the bound
|
||||
# is unused on that path.
|
||||
os.environ["AITER_BF16_FP8_MOE_BOUND"] = "0"
|
||||
|
||||
from aiter.ops.shuffle import shuffle_scale as _shuf_s
|
||||
from aiter.ops.shuffle import shuffle_weight as _shuf_w
|
||||
|
||||
|
||||
Reference in New Issue
Block a user