[ROCm][Kimi-K3] Make VLLM_ROCM_USE_AITER_MOE_SITUV2 select a4w4/a8w4/a16w4 (#58201)

Signed-off-by: Hongxia Yang <hongxia.yang@amd.com>
This commit is contained in:
Hongxia Yang
2026-09-27 02:26:49 +00:00
committed by GitHub
parent 0c87a197b8
commit 3137ff0773
5 changed files with 109 additions and 94 deletions
+44 -52
View File
@@ -566,72 +566,64 @@ def test_aiter_moe_shared_experts_enablement_follows_env(
assert rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() is expected
def test_aiter_moe_situv2_syncs_aiter_a4w4_env(
monkeypatch: pytest.MonkeyPatch,
):
"""SiTUv2 MoE should route to AITER a4w4 and clear legacy a8w4 overrides."""
from vllm._aiter_ops import rocm_aiter_ops
_assert_aiter_supported()
with monkeypatch.context() as mp:
mp.delenv("AITER_SITUV2_A8W4", raising=False)
mp.delenv("AITER_SITUV2_A4W4", raising=False)
mp.setenv("VLLM_ROCM_USE_AITER", "1")
mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "1")
_reload_envs()
rocm_aiter_ops.refresh_env_variables()
import os
assert os.environ.get("AITER_SITUV2_A4W4") == "1"
assert "AITER_SITUV2_A8W4" not in os.environ
def test_aiter_moe_situv2_legacy_a8w4_alias_enables_a4w4(
monkeypatch: pytest.MonkeyPatch,
):
"""Deprecated VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4 still enables SiTUv2 a4w4."""
from vllm._aiter_ops import rocm_aiter_ops
_assert_aiter_supported()
with monkeypatch.context() as mp:
mp.delenv("AITER_SITUV2_A8W4", raising=False)
mp.delenv("AITER_SITUV2_A4W4", raising=False)
mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", raising=False)
mp.setenv("VLLM_ROCM_USE_AITER", "1")
mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", "1")
_reload_envs()
rocm_aiter_ops.refresh_env_variables()
import os
assert os.environ.get("AITER_SITUV2_A4W4") == "1"
assert "AITER_SITUV2_A8W4" not in os.environ
def test_aiter_moe_situv2_clears_aiter_env_when_disabled(
@pytest.mark.parametrize(
"value,expected_act,expected_env",
[
# Unset / auto / legacy "1" all mean the a4w4 default.
(None, "a4w4", {"AITER_SITUV2_A4W4": "1"}),
("auto", "a4w4", {"AITER_SITUV2_A4W4": "1"}),
("1", "a4w4", {"AITER_SITUV2_A4W4": "1"}),
("a4w4", "a4w4", {"AITER_SITUV2_A4W4": "1"}),
("0", "a16w4", {}),
("a16w4", "a16w4", {}),
("A8W4", "a8w4", {"AITER_SITUV2_A8W4": "1"}),
],
)
def test_aiter_moe_situv2_activation_syncs_aiter_env(
value: str | None,
expected_act: str,
expected_env: dict[str, str],
monkeypatch: pytest.MonkeyPatch,
):
"""Each SiTUv2 activation choice sets exactly its AITER dispatch env."""
import os
from vllm._aiter_ops import rocm_aiter_ops
_assert_aiter_supported()
with monkeypatch.context() as mp:
# Stale AITER overrides must be replaced by the vLLM selection.
mp.setenv("AITER_SITUV2_A8W4", "1")
mp.setenv("AITER_SITUV2_A4W4", "1")
mp.setenv("VLLM_ROCM_USE_AITER", "1")
mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", raising=False)
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "0")
if value is None:
mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", raising=False)
else:
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", value)
_reload_envs()
rocm_aiter_ops.refresh_env_variables()
import os
assert rocm_aiter_ops.get_fused_moe_situv2_activation() == expected_act
assert rocm_aiter_ops.is_fused_moe_situv2_gate_up_interleaved() is (
expected_act == "a8w4"
)
assert rocm_aiter_ops.is_fused_moe_situv2_enabled() is (expected_act != "a16w4")
for name in ("AITER_SITUV2_A8W4", "AITER_SITUV2_A4W4"):
assert os.environ.get(name) == expected_env.get(name)
assert "AITER_SITUV2_A4W4" not in os.environ
def test_aiter_moe_situv2_rejects_unknown_activation(
monkeypatch: pytest.MonkeyPatch,
):
with monkeypatch.context() as mp:
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "a8w8")
_reload_envs()
import vllm.envs as envs
with pytest.raises(ValueError, match="VLLM_ROCM_USE_AITER_MOE_SITUV2"):
_ = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2
@pytest.mark.parametrize("moe_padding", [True, False])
+51 -23
View File
@@ -1870,28 +1870,39 @@ def _mhc_delayed_pre_tail(
_OPS_REGISTERED = False
_AITER_SITUV2_ACT_ENV = {
"a8w4": "AITER_SITUV2_A8W4",
"a4w4": "AITER_SITUV2_A4W4",
}
def _resolve_situv2_activation() -> str:
"""VLLM_ROCM_USE_AITER_MOE_SITUV2 -> a4w4 | a8w4 | a16w4.
auto (default) and the legacy 1 mean a4w4; legacy 0 means a16w4.
"""
value = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2.lower()
if value in ("auto", "1"):
return "a4w4"
if value == "0":
return "a16w4"
return value
def _sync_aiter_situv2_moe_env() -> None:
"""Mirror the SiTUv2 MoE toggle into AITER's a4w4 dispatch env.
"""Set the AITER_SITUV2_* env matching VLLM_ROCM_USE_AITER_MOE_SITUV2.
AITER selects afp8 vs afp4 activation kernels via AITER_SITUV2_A8W4 /
AITER_SITUV2_A4W4 (see ROCm/aiter fused_moe.py, A8W4 checked first).
When VLLM_ROCM_USE_AITER_MOE_SITUV2 is enabled we route to a4w4
(afp4_wfp4_fp4 kernels) and clear any legacy AITER_SITUV2_A8W4 override.
Requires AITER with ROCm/aiter#4463 (first tagged in v0.1.20): a4w4
dispatch plus kimik3_a4w4_{un,}tuned_fmoe.csv. Older AITER still runs
afp4 FlyDSL but falls back to heuristic configs, which is not the
tuned a4w4 path this recipe is meant to use.
AITER reads AITER_SITUV2_A8W4 / AITER_SITUV2_A4W4 and defaults to a16w4,
so exactly one is set for a8w4/a4w4 and both are cleared for a16w4.
"""
import os
import vllm.envs as envs
if envs.VLLM_ROCM_USE_AITER_MOE_SITUV2:
os.environ["AITER_SITUV2_A4W4"] = "1"
os.environ.pop("AITER_SITUV2_A8W4", None)
else:
os.environ.pop("AITER_SITUV2_A4W4", None)
selected = _AITER_SITUV2_ACT_ENV.get(_resolve_situv2_activation())
for name in _AITER_SITUV2_ACT_ENV.values():
if name == selected:
os.environ[name] = "1"
else:
os.environ.pop(name, None)
class rocm_aiter_ops:
@@ -1917,7 +1928,8 @@ class rocm_aiter_ops:
VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: Controls FP4 assembly GEMM.
VLLM_ROCM_USE_AITER_TRITON_ROPE: Controls Triton rotary embeddings.
VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: Controls shared expert fusion.
VLLM_ROCM_USE_AITER_MOE_SITUV2: Controls SiTUv2 FlyDSL MoE (a4w4).
VLLM_ROCM_USE_AITER_MOE_SITUV2: SiTUv2 FlyDSL MoE activation
dtype (a16w4 | a8w4 | a4w4).
VLLM_ROCM_USE_AITER_TRITON_GEMM: Controls Triton unquantized GEMM.
Note:
@@ -1987,7 +1999,7 @@ class rocm_aiter_ops:
# TODO: Consolidate under VLLM_ROCM_USE_AITER_ROPE
_TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE
_MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
_MOE_SITUV2 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2
_MOE_SITUV2 = _resolve_situv2_activation()
# TODO: Consolidate under _LINEAR_ENABLED
_TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM
# Lazily probed: whether aiter.topk_softmax supports the
@@ -2016,7 +2028,7 @@ class rocm_aiter_ops:
cls._FP4_GEMM_DYNAMIC_QUANT_ASM = envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM
cls._TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE
cls._MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
cls._MOE_SITUV2 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2
cls._MOE_SITUV2 = _resolve_situv2_activation()
_sync_aiter_situv2_moe_env()
cls._TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM
cls._MOE_DISPATCH_POLICY = envs.VLLM_ROCM_AITER_MOE_DISPATCH_POLICY
@@ -2132,9 +2144,25 @@ class rocm_aiter_ops:
@classmethod
@if_aiter_supported
def is_fused_moe_situv2_enabled(cls) -> bool:
# _MOE_SITUV2 is a variant of aiter fused moe, so aiter
# fused moe must be enabled as well.
return cls.is_fused_moe_enabled() and cls._MOE_SITUV2
"""True when a low-precision (a8w4/a4w4) SiTUv2 activation is selected."""
return cls.is_fused_moe_enabled() and cls._MOE_SITUV2 != "a16w4"
@classmethod
@if_aiter_supported
def get_fused_moe_situv2_activation(cls) -> str:
"""SiTUv2 activation dtype name: a16w4, a8w4 or a4w4."""
return cls._MOE_SITUV2
@classmethod
@if_aiter_supported
def is_fused_moe_situv2_gate_up_interleaved(cls) -> bool:
"""Whether the SiTUv2 kernels expect gate/up-interleaved w13.
The a8w4 FlyDSL stage-1 kernels (_gui_) are interleaved; a16w4 and
a4w4 are separated. Weight shuffle (oracle/mxfp4.py) and gate_mode
(experts/rocm_aiter_moe.py) must both use this.
"""
return cls._MOE_SITUV2 == "a8w4"
@classmethod
@if_aiter_supported
+8 -13
View File
@@ -141,7 +141,7 @@ if TYPE_CHECKING:
VLLM_ROCM_USE_AITER_LINEAR_HIPBMM: bool = False
VLLM_ROCM_USE_AITER_MOE: bool = True
VLLM_ROCM_AITER_MOE_DISPATCH_POLICY: int = 0
VLLM_ROCM_USE_AITER_MOE_SITUV2: bool = False
VLLM_ROCM_USE_AITER_MOE_SITUV2: Literal["auto", "a4w4", "a8w4", "a16w4"] = "auto"
VLLM_ROCM_USE_AITER_RMSNORM: bool = True
VLLM_ROCM_USE_AITER_MLA: bool = True
VLLM_ROCM_AITER_MLA_ASM_PADDING: Literal["auto", "gluon", "asm"] = "auto"
@@ -1280,18 +1280,13 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_ROCM_USE_AITER_MOE": lambda: (
os.getenv("VLLM_ROCM_USE_AITER_MOE", "True").lower() in ("true", "1")
),
# Route K3 SiTU MXFP4 MoE through the FlyDSL SiTUv2 path (a4w4 fp4
# activations, separated gate/up layout) instead of default a16w4. vLLM
# sets AITER_SITUV2_A4W4 at init when this flag is on and clears any
# legacy AITER_SITUV2_A8W4 override (AITER checks A8W4 first).
# VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4 is a deprecated alias for existing
# recipes; it does not select a8w4 kernels.
# Needs AITER >= v0.1.20 (ROCm/aiter#4463) for the a4w4 dispatch flag
# and tuned kimik3_a4w4_*_fmoe.csv rows; otherwise FlyDSL uses heuristics.
"VLLM_ROCM_USE_AITER_MOE_SITUV2": lambda: (
os.getenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "0").lower() in ("true", "1")
or os.getenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", "0").lower()
in ("true", "1")
# Activation dtype for the Kimi-K3 SiTU MXFP4 MoE (AITER FlyDSL SiTUv2):
# auto (= a4w4), a4w4, a8w4 or a16w4. Legacy 1/0 mean a4w4/a16w4.
"VLLM_ROCM_USE_AITER_MOE_SITUV2": env_with_choices(
"VLLM_ROCM_USE_AITER_MOE_SITUV2",
"auto",
["auto", "a4w4", "a8w4", "a16w4", "0", "1"],
case_sensitive=False,
),
# MoE sorting dispatch policy for AITER fused MoE kernels.
# 0 = auto (default): single-pass for small batches, multi-pass
@@ -385,10 +385,11 @@ def rocm_aiter_fused_experts(
gate_mode = ""
if activation == MoEActivation.SITU:
# SiTUv2 flydsl (VLLM_ROCM_USE_AITER_MOE_SITUV2=1) uses a4w4
# fp4 activations with separated gate/up weights (AITER #4463);
# default a16w4 SiTU also stays separated.
gate_mode = GateMode.SEPARATED.value
gate_mode = (
GateMode.INTERLEAVE.value
if rocm_aiter_ops.is_fused_moe_situv2_gate_up_interleaved()
else GateMode.SEPARATED.value
)
elif quant_config.use_mxfp4_w4a16:
gate_mode = GateMode.INTERLEAVE.value
elif activation_interleave is not None:
@@ -1596,8 +1596,7 @@ def convert_weight_to_mxfp4_moe_kernel_format(
fp4_dtype = torch.float4_e2m1fn_x2
e8m0_dtype = torch.float8_e8m0fnu
# SiTUv2 flydsl uses separated gate/up layout (a4w4).
guinterleave = False
guinterleave = rocm_aiter_ops.is_fused_moe_situv2_gate_up_interleaved()
w13 = rocm_aiter_ops.shuffle_weight_a16w4(
w13_weight.data.view(fp4_dtype), 16, guinterleave
)