[ROCm][Kimi-K3] Make VLLM_ROCM_USE_AITER_MOE_SITUV2 select a4w4/a8w4/a16w4 (#58201)
Signed-off-by: Hongxia Yang <hongxia.yang@amd.com>
This commit is contained in:
@@ -566,72 +566,64 @@ def test_aiter_moe_shared_experts_enablement_follows_env(
|
||||
assert rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() is expected
|
||||
|
||||
|
||||
def test_aiter_moe_situv2_syncs_aiter_a4w4_env(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""SiTUv2 MoE should route to AITER a4w4 and clear legacy a8w4 overrides."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
_assert_aiter_supported()
|
||||
|
||||
with monkeypatch.context() as mp:
|
||||
mp.delenv("AITER_SITUV2_A8W4", raising=False)
|
||||
mp.delenv("AITER_SITUV2_A4W4", raising=False)
|
||||
mp.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "1")
|
||||
_reload_envs()
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
import os
|
||||
|
||||
assert os.environ.get("AITER_SITUV2_A4W4") == "1"
|
||||
assert "AITER_SITUV2_A8W4" not in os.environ
|
||||
|
||||
|
||||
def test_aiter_moe_situv2_legacy_a8w4_alias_enables_a4w4(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Deprecated VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4 still enables SiTUv2 a4w4."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
_assert_aiter_supported()
|
||||
|
||||
with monkeypatch.context() as mp:
|
||||
mp.delenv("AITER_SITUV2_A8W4", raising=False)
|
||||
mp.delenv("AITER_SITUV2_A4W4", raising=False)
|
||||
mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", raising=False)
|
||||
mp.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", "1")
|
||||
_reload_envs()
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
import os
|
||||
|
||||
assert os.environ.get("AITER_SITUV2_A4W4") == "1"
|
||||
assert "AITER_SITUV2_A8W4" not in os.environ
|
||||
|
||||
|
||||
def test_aiter_moe_situv2_clears_aiter_env_when_disabled(
|
||||
@pytest.mark.parametrize(
|
||||
"value,expected_act,expected_env",
|
||||
[
|
||||
# Unset / auto / legacy "1" all mean the a4w4 default.
|
||||
(None, "a4w4", {"AITER_SITUV2_A4W4": "1"}),
|
||||
("auto", "a4w4", {"AITER_SITUV2_A4W4": "1"}),
|
||||
("1", "a4w4", {"AITER_SITUV2_A4W4": "1"}),
|
||||
("a4w4", "a4w4", {"AITER_SITUV2_A4W4": "1"}),
|
||||
("0", "a16w4", {}),
|
||||
("a16w4", "a16w4", {}),
|
||||
("A8W4", "a8w4", {"AITER_SITUV2_A8W4": "1"}),
|
||||
],
|
||||
)
|
||||
def test_aiter_moe_situv2_activation_syncs_aiter_env(
|
||||
value: str | None,
|
||||
expected_act: str,
|
||||
expected_env: dict[str, str],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Each SiTUv2 activation choice sets exactly its AITER dispatch env."""
|
||||
import os
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
_assert_aiter_supported()
|
||||
|
||||
with monkeypatch.context() as mp:
|
||||
# Stale AITER overrides must be replaced by the vLLM selection.
|
||||
mp.setenv("AITER_SITUV2_A8W4", "1")
|
||||
mp.setenv("AITER_SITUV2_A4W4", "1")
|
||||
mp.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
|
||||
mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", raising=False)
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "0")
|
||||
if value is None:
|
||||
mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", raising=False)
|
||||
else:
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", value)
|
||||
_reload_envs()
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
import os
|
||||
assert rocm_aiter_ops.get_fused_moe_situv2_activation() == expected_act
|
||||
assert rocm_aiter_ops.is_fused_moe_situv2_gate_up_interleaved() is (
|
||||
expected_act == "a8w4"
|
||||
)
|
||||
assert rocm_aiter_ops.is_fused_moe_situv2_enabled() is (expected_act != "a16w4")
|
||||
for name in ("AITER_SITUV2_A8W4", "AITER_SITUV2_A4W4"):
|
||||
assert os.environ.get(name) == expected_env.get(name)
|
||||
|
||||
assert "AITER_SITUV2_A4W4" not in os.environ
|
||||
|
||||
def test_aiter_moe_situv2_rejects_unknown_activation(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
with monkeypatch.context() as mp:
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "a8w8")
|
||||
_reload_envs()
|
||||
import vllm.envs as envs
|
||||
|
||||
with pytest.raises(ValueError, match="VLLM_ROCM_USE_AITER_MOE_SITUV2"):
|
||||
_ = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("moe_padding", [True, False])
|
||||
|
||||
+51
-23
@@ -1870,28 +1870,39 @@ def _mhc_delayed_pre_tail(
|
||||
_OPS_REGISTERED = False
|
||||
|
||||
|
||||
_AITER_SITUV2_ACT_ENV = {
|
||||
"a8w4": "AITER_SITUV2_A8W4",
|
||||
"a4w4": "AITER_SITUV2_A4W4",
|
||||
}
|
||||
|
||||
|
||||
def _resolve_situv2_activation() -> str:
|
||||
"""VLLM_ROCM_USE_AITER_MOE_SITUV2 -> a4w4 | a8w4 | a16w4.
|
||||
|
||||
auto (default) and the legacy 1 mean a4w4; legacy 0 means a16w4.
|
||||
"""
|
||||
value = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2.lower()
|
||||
if value in ("auto", "1"):
|
||||
return "a4w4"
|
||||
if value == "0":
|
||||
return "a16w4"
|
||||
return value
|
||||
|
||||
|
||||
def _sync_aiter_situv2_moe_env() -> None:
|
||||
"""Mirror the SiTUv2 MoE toggle into AITER's a4w4 dispatch env.
|
||||
"""Set the AITER_SITUV2_* env matching VLLM_ROCM_USE_AITER_MOE_SITUV2.
|
||||
|
||||
AITER selects afp8 vs afp4 activation kernels via AITER_SITUV2_A8W4 /
|
||||
AITER_SITUV2_A4W4 (see ROCm/aiter fused_moe.py, A8W4 checked first).
|
||||
When VLLM_ROCM_USE_AITER_MOE_SITUV2 is enabled we route to a4w4
|
||||
(afp4_wfp4_fp4 kernels) and clear any legacy AITER_SITUV2_A8W4 override.
|
||||
|
||||
Requires AITER with ROCm/aiter#4463 (first tagged in v0.1.20): a4w4
|
||||
dispatch plus kimik3_a4w4_{un,}tuned_fmoe.csv. Older AITER still runs
|
||||
afp4 FlyDSL but falls back to heuristic configs, which is not the
|
||||
tuned a4w4 path this recipe is meant to use.
|
||||
AITER reads AITER_SITUV2_A8W4 / AITER_SITUV2_A4W4 and defaults to a16w4,
|
||||
so exactly one is set for a8w4/a4w4 and both are cleared for a16w4.
|
||||
"""
|
||||
import os
|
||||
|
||||
import vllm.envs as envs
|
||||
|
||||
if envs.VLLM_ROCM_USE_AITER_MOE_SITUV2:
|
||||
os.environ["AITER_SITUV2_A4W4"] = "1"
|
||||
os.environ.pop("AITER_SITUV2_A8W4", None)
|
||||
else:
|
||||
os.environ.pop("AITER_SITUV2_A4W4", None)
|
||||
selected = _AITER_SITUV2_ACT_ENV.get(_resolve_situv2_activation())
|
||||
for name in _AITER_SITUV2_ACT_ENV.values():
|
||||
if name == selected:
|
||||
os.environ[name] = "1"
|
||||
else:
|
||||
os.environ.pop(name, None)
|
||||
|
||||
|
||||
class rocm_aiter_ops:
|
||||
@@ -1917,7 +1928,8 @@ class rocm_aiter_ops:
|
||||
VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: Controls FP4 assembly GEMM.
|
||||
VLLM_ROCM_USE_AITER_TRITON_ROPE: Controls Triton rotary embeddings.
|
||||
VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: Controls shared expert fusion.
|
||||
VLLM_ROCM_USE_AITER_MOE_SITUV2: Controls SiTUv2 FlyDSL MoE (a4w4).
|
||||
VLLM_ROCM_USE_AITER_MOE_SITUV2: SiTUv2 FlyDSL MoE activation
|
||||
dtype (a16w4 | a8w4 | a4w4).
|
||||
VLLM_ROCM_USE_AITER_TRITON_GEMM: Controls Triton unquantized GEMM.
|
||||
|
||||
Note:
|
||||
@@ -1987,7 +1999,7 @@ class rocm_aiter_ops:
|
||||
# TODO: Consolidate under VLLM_ROCM_USE_AITER_ROPE
|
||||
_TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE
|
||||
_MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
|
||||
_MOE_SITUV2 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2
|
||||
_MOE_SITUV2 = _resolve_situv2_activation()
|
||||
# TODO: Consolidate under _LINEAR_ENABLED
|
||||
_TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM
|
||||
# Lazily probed: whether aiter.topk_softmax supports the
|
||||
@@ -2016,7 +2028,7 @@ class rocm_aiter_ops:
|
||||
cls._FP4_GEMM_DYNAMIC_QUANT_ASM = envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM
|
||||
cls._TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE
|
||||
cls._MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
|
||||
cls._MOE_SITUV2 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2
|
||||
cls._MOE_SITUV2 = _resolve_situv2_activation()
|
||||
_sync_aiter_situv2_moe_env()
|
||||
cls._TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM
|
||||
cls._MOE_DISPATCH_POLICY = envs.VLLM_ROCM_AITER_MOE_DISPATCH_POLICY
|
||||
@@ -2132,9 +2144,25 @@ class rocm_aiter_ops:
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
def is_fused_moe_situv2_enabled(cls) -> bool:
|
||||
# _MOE_SITUV2 is a variant of aiter fused moe, so aiter
|
||||
# fused moe must be enabled as well.
|
||||
return cls.is_fused_moe_enabled() and cls._MOE_SITUV2
|
||||
"""True when a low-precision (a8w4/a4w4) SiTUv2 activation is selected."""
|
||||
return cls.is_fused_moe_enabled() and cls._MOE_SITUV2 != "a16w4"
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
def get_fused_moe_situv2_activation(cls) -> str:
|
||||
"""SiTUv2 activation dtype name: a16w4, a8w4 or a4w4."""
|
||||
return cls._MOE_SITUV2
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
def is_fused_moe_situv2_gate_up_interleaved(cls) -> bool:
|
||||
"""Whether the SiTUv2 kernels expect gate/up-interleaved w13.
|
||||
|
||||
The a8w4 FlyDSL stage-1 kernels (_gui_) are interleaved; a16w4 and
|
||||
a4w4 are separated. Weight shuffle (oracle/mxfp4.py) and gate_mode
|
||||
(experts/rocm_aiter_moe.py) must both use this.
|
||||
"""
|
||||
return cls._MOE_SITUV2 == "a8w4"
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
|
||||
+8
-13
@@ -141,7 +141,7 @@ if TYPE_CHECKING:
|
||||
VLLM_ROCM_USE_AITER_LINEAR_HIPBMM: bool = False
|
||||
VLLM_ROCM_USE_AITER_MOE: bool = True
|
||||
VLLM_ROCM_AITER_MOE_DISPATCH_POLICY: int = 0
|
||||
VLLM_ROCM_USE_AITER_MOE_SITUV2: bool = False
|
||||
VLLM_ROCM_USE_AITER_MOE_SITUV2: Literal["auto", "a4w4", "a8w4", "a16w4"] = "auto"
|
||||
VLLM_ROCM_USE_AITER_RMSNORM: bool = True
|
||||
VLLM_ROCM_USE_AITER_MLA: bool = True
|
||||
VLLM_ROCM_AITER_MLA_ASM_PADDING: Literal["auto", "gluon", "asm"] = "auto"
|
||||
@@ -1280,18 +1280,13 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_ROCM_USE_AITER_MOE": lambda: (
|
||||
os.getenv("VLLM_ROCM_USE_AITER_MOE", "True").lower() in ("true", "1")
|
||||
),
|
||||
# Route K3 SiTU MXFP4 MoE through the FlyDSL SiTUv2 path (a4w4 fp4
|
||||
# activations, separated gate/up layout) instead of default a16w4. vLLM
|
||||
# sets AITER_SITUV2_A4W4 at init when this flag is on and clears any
|
||||
# legacy AITER_SITUV2_A8W4 override (AITER checks A8W4 first).
|
||||
# VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4 is a deprecated alias for existing
|
||||
# recipes; it does not select a8w4 kernels.
|
||||
# Needs AITER >= v0.1.20 (ROCm/aiter#4463) for the a4w4 dispatch flag
|
||||
# and tuned kimik3_a4w4_*_fmoe.csv rows; otherwise FlyDSL uses heuristics.
|
||||
"VLLM_ROCM_USE_AITER_MOE_SITUV2": lambda: (
|
||||
os.getenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "0").lower() in ("true", "1")
|
||||
or os.getenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", "0").lower()
|
||||
in ("true", "1")
|
||||
# Activation dtype for the Kimi-K3 SiTU MXFP4 MoE (AITER FlyDSL SiTUv2):
|
||||
# auto (= a4w4), a4w4, a8w4 or a16w4. Legacy 1/0 mean a4w4/a16w4.
|
||||
"VLLM_ROCM_USE_AITER_MOE_SITUV2": env_with_choices(
|
||||
"VLLM_ROCM_USE_AITER_MOE_SITUV2",
|
||||
"auto",
|
||||
["auto", "a4w4", "a8w4", "a16w4", "0", "1"],
|
||||
case_sensitive=False,
|
||||
),
|
||||
# MoE sorting dispatch policy for AITER fused MoE kernels.
|
||||
# 0 = auto (default): single-pass for small batches, multi-pass
|
||||
|
||||
@@ -385,10 +385,11 @@ def rocm_aiter_fused_experts(
|
||||
|
||||
gate_mode = ""
|
||||
if activation == MoEActivation.SITU:
|
||||
# SiTUv2 flydsl (VLLM_ROCM_USE_AITER_MOE_SITUV2=1) uses a4w4
|
||||
# fp4 activations with separated gate/up weights (AITER #4463);
|
||||
# default a16w4 SiTU also stays separated.
|
||||
gate_mode = GateMode.SEPARATED.value
|
||||
gate_mode = (
|
||||
GateMode.INTERLEAVE.value
|
||||
if rocm_aiter_ops.is_fused_moe_situv2_gate_up_interleaved()
|
||||
else GateMode.SEPARATED.value
|
||||
)
|
||||
elif quant_config.use_mxfp4_w4a16:
|
||||
gate_mode = GateMode.INTERLEAVE.value
|
||||
elif activation_interleave is not None:
|
||||
|
||||
@@ -1596,8 +1596,7 @@ def convert_weight_to_mxfp4_moe_kernel_format(
|
||||
|
||||
fp4_dtype = torch.float4_e2m1fn_x2
|
||||
e8m0_dtype = torch.float8_e8m0fnu
|
||||
# SiTUv2 flydsl uses separated gate/up layout (a4w4).
|
||||
guinterleave = False
|
||||
guinterleave = rocm_aiter_ops.is_fused_moe_situv2_gate_up_interleaved()
|
||||
w13 = rocm_aiter_ops.shuffle_weight_a16w4(
|
||||
w13_weight.data.view(fp4_dtype), 16, guinterleave
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user