[ROCm][Kimi-K3] Enable a4w4 flydsl kernels for KimiK3 (#53940)

Signed-off-by: ppalanga <ppalanga@amd.com>
Signed-off-by: Poovaiah Palangappa <poovaiah.palangappa@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Poovaiah Palangappa <poovaiah.palangappa@amd.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Hongxia Yang <62075498+hongxiayang@users.noreply.github.com>
This commit is contained in:
ppalanga
2026-09-16 01:38:02 +00:00
committed by GitHub
co-authored by Cursor Poovaiah Palangappa mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Hongxia Yang
parent c8d1cf077a
commit a4d2d9d95e
5 changed files with 127 additions and 30 deletions
+32 -6
View File
@@ -1752,6 +1752,30 @@ def _mhc_delayed_pre_tail(
_OPS_REGISTERED = False
def _sync_aiter_situv2_moe_env() -> None:
"""Mirror the SiTUv2 MoE toggle into AITER's a4w4 dispatch env.
AITER selects afp8 vs afp4 activation kernels via AITER_SITUV2_A8W4 /
AITER_SITUV2_A4W4 (see ROCm/aiter fused_moe.py, A8W4 checked first).
When VLLM_ROCM_USE_AITER_MOE_SITUV2 is enabled we route to a4w4
(afp4_wfp4_fp4 kernels) and clear any legacy AITER_SITUV2_A8W4 override.
Requires AITER with ROCm/aiter#4463 (first tagged in v0.1.20): a4w4
dispatch plus kimik3_a4w4_{un,}tuned_fmoe.csv. Older AITER still runs
afp4 FlyDSL but falls back to heuristic configs, which is not the
tuned a4w4 path this recipe is meant to use.
"""
import os
import vllm.envs as envs
if envs.VLLM_ROCM_USE_AITER_MOE_SITUV2:
os.environ["AITER_SITUV2_A4W4"] = "1"
os.environ.pop("AITER_SITUV2_A8W4", None)
else:
os.environ.pop("AITER_SITUV2_A4W4", None)
class rocm_aiter_ops:
"""ROCm AITER operations wrapper for AMD GPU acceleration in vLLM.
@@ -1774,7 +1798,7 @@ class rocm_aiter_ops:
VLLM_ROCM_USE_AITER_FP8BMM: Controls FP8 batched matrix multiply.
VLLM_ROCM_USE_AITER_TRITON_ROPE: Controls Triton rotary embeddings.
VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: Controls shared expert fusion.
VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4: Controls a8w4 SiTU fused MoE variant.
VLLM_ROCM_USE_AITER_MOE_SITUV2: Controls SiTUv2 FlyDSL MoE (a4w4).
VLLM_ROCM_USE_AITER_TRITON_GEMM: Controls Triton unquantized GEMM.
Note:
@@ -1841,7 +1865,7 @@ class rocm_aiter_ops:
# TODO: Consolidate under VLLM_ROCM_USE_AITER_ROPE
_TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE
_MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
_MOE_SITUV2_A8W4 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4
_MOE_SITUV2 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2
# TODO: Consolidate under _LINEAR_ENABLED
_TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM
# Lazily probed: whether aiter.topk_softmax supports the
@@ -1870,7 +1894,8 @@ class rocm_aiter_ops:
cls._LINEAR_HIPBMM_ENABLED = envs.VLLM_ROCM_USE_AITER_LINEAR_HIPBMM
cls._TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE
cls._MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
cls._MOE_SITUV2_A8W4 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4
cls._MOE_SITUV2 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2
_sync_aiter_situv2_moe_env()
cls._TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM
@staticmethod
@@ -1983,10 +2008,10 @@ class rocm_aiter_ops:
@classmethod
@if_aiter_supported
def is_fused_moe_situv2_a8w4_enabled(cls) -> bool:
# _MOE_SITUV2_A8W4 is a variant of aiter fused moe, so aiter
def is_fused_moe_situv2_enabled(cls) -> bool:
# _MOE_SITUV2 is a variant of aiter fused moe, so aiter
# fused moe must be enabled as well.
return cls.is_fused_moe_enabled() and cls._MOE_SITUV2_A8W4
return cls.is_fused_moe_enabled() and cls._MOE_SITUV2
@classmethod
@if_aiter_supported
@@ -4029,3 +4054,4 @@ class rocm_aiter_ops:
rocm_aiter_ops.register_ops_once()
_sync_aiter_situv2_moe_env()