[ROCm][Kimi-K3] Enable a4w4 flydsl kernels for KimiK3 (#53940)
Signed-off-by: ppalanga <ppalanga@amd.com> Signed-off-by: Poovaiah Palangappa <poovaiah.palangappa@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Poovaiah Palangappa <poovaiah.palangappa@amd.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Hongxia Yang <62075498+hongxiayang@users.noreply.github.com>
This commit is contained in:
co-authored by
Cursor
Poovaiah Palangappa
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Hongxia Yang
parent
c8d1cf077a
commit
a4d2d9d95e
+13
-7
@@ -137,7 +137,7 @@ if TYPE_CHECKING:
|
||||
VLLM_ROCM_USE_AITER_LINEAR_HIPBMM: bool = False
|
||||
VLLM_ROCM_USE_AITER_MOE: bool = True
|
||||
VLLM_ROCM_AITER_MOE_DISPATCH_POLICY: int = 0
|
||||
VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4: bool = False
|
||||
VLLM_ROCM_USE_AITER_MOE_SITUV2: bool = False
|
||||
VLLM_ROCM_USE_AITER_RMSNORM: bool = True
|
||||
VLLM_ROCM_USE_AITER_MLA: bool = True
|
||||
VLLM_ROCM_AITER_MLA_ASM_PADDING: Literal["auto", "gluon", "asm"] = "auto"
|
||||
@@ -1251,12 +1251,18 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_ROCM_USE_AITER_MOE": lambda: (
|
||||
os.getenv("VLLM_ROCM_USE_AITER_MOE", "True").lower() in ("true", "1")
|
||||
),
|
||||
# Route K3 SiTU MXFP4 MoE through the a8w4 (fp8 activation) gate/up-
|
||||
# interleaved flydsl kernels instead of the default a16w4 separated path.
|
||||
# This is the only flag users need: vLLM picks the kernels by passing
|
||||
# gate_mode to AITER and sets the AITER-side workaround env at init.
|
||||
"VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4": lambda: (
|
||||
os.getenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", "0").lower() in ("true", "1")
|
||||
# Route K3 SiTU MXFP4 MoE through the FlyDSL SiTUv2 path (a4w4 fp4
|
||||
# activations, separated gate/up layout) instead of default a16w4. vLLM
|
||||
# sets AITER_SITUV2_A4W4 at init when this flag is on and clears any
|
||||
# legacy AITER_SITUV2_A8W4 override (AITER checks A8W4 first).
|
||||
# VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4 is a deprecated alias for existing
|
||||
# recipes; it does not select a8w4 kernels.
|
||||
# Needs AITER >= v0.1.20 (ROCm/aiter#4463) for the a4w4 dispatch flag
|
||||
# and tuned kimik3_a4w4_*_fmoe.csv rows; otherwise FlyDSL uses heuristics.
|
||||
"VLLM_ROCM_USE_AITER_MOE_SITUV2": lambda: (
|
||||
os.getenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "0").lower() in ("true", "1")
|
||||
or os.getenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", "0").lower()
|
||||
in ("true", "1")
|
||||
),
|
||||
# MoE sorting dispatch policy for AITER fused MoE kernels.
|
||||
# 0 = auto (default): single-pass for small batches, multi-pass
|
||||
|
||||
Reference in New Issue
Block a user