[ROCm][Kimi-K3] Enable a4w4 flydsl kernels for KimiK3 (#53940)

Signed-off-by: ppalanga <ppalanga@amd.com>
Signed-off-by: Poovaiah Palangappa <poovaiah.palangappa@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Poovaiah Palangappa <poovaiah.palangappa@amd.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Hongxia Yang <62075498+hongxiayang@users.noreply.github.com>
This commit is contained in:
ppalanga
2026-09-16 01:38:02 +00:00
committed by GitHub
co-authored by Cursor Poovaiah Palangappa mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Hongxia Yang
parent c8d1cf077a
commit a4d2d9d95e
5 changed files with 127 additions and 30 deletions
+13 -7
View File
@@ -137,7 +137,7 @@ if TYPE_CHECKING:
VLLM_ROCM_USE_AITER_LINEAR_HIPBMM: bool = False
VLLM_ROCM_USE_AITER_MOE: bool = True
VLLM_ROCM_AITER_MOE_DISPATCH_POLICY: int = 0
VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4: bool = False
VLLM_ROCM_USE_AITER_MOE_SITUV2: bool = False
VLLM_ROCM_USE_AITER_RMSNORM: bool = True
VLLM_ROCM_USE_AITER_MLA: bool = True
VLLM_ROCM_AITER_MLA_ASM_PADDING: Literal["auto", "gluon", "asm"] = "auto"
@@ -1251,12 +1251,18 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_ROCM_USE_AITER_MOE": lambda: (
os.getenv("VLLM_ROCM_USE_AITER_MOE", "True").lower() in ("true", "1")
),
# Route K3 SiTU MXFP4 MoE through the a8w4 (fp8 activation) gate/up-
# interleaved flydsl kernels instead of the default a16w4 separated path.
# This is the only flag users need: vLLM picks the kernels by passing
# gate_mode to AITER and sets the AITER-side workaround env at init.
"VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4": lambda: (
os.getenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", "0").lower() in ("true", "1")
# Route K3 SiTU MXFP4 MoE through the FlyDSL SiTUv2 path (a4w4 fp4
# activations, separated gate/up layout) instead of default a16w4. vLLM
# sets AITER_SITUV2_A4W4 at init when this flag is on and clears any
# legacy AITER_SITUV2_A8W4 override (AITER checks A8W4 first).
# VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4 is a deprecated alias for existing
# recipes; it does not select a8w4 kernels.
# Needs AITER >= v0.1.20 (ROCm/aiter#4463) for the a4w4 dispatch flag
# and tuned kimik3_a4w4_*_fmoe.csv rows; otherwise FlyDSL uses heuristics.
"VLLM_ROCM_USE_AITER_MOE_SITUV2": lambda: (
os.getenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "0").lower() in ("true", "1")
or os.getenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", "0").lower()
in ("true", "1")
),
# MoE sorting dispatch policy for AITER fused MoE kernels.
# 0 = auto (default): single-pass for small batches, multi-pass