diff --git a/tests/kernels/moe/test_rocm_aiter_moe.py b/tests/kernels/moe/test_rocm_aiter_moe.py index 8fa1706f18..fbc8586fd3 100644 --- a/tests/kernels/moe/test_rocm_aiter_moe.py +++ b/tests/kernels/moe/test_rocm_aiter_moe.py @@ -566,72 +566,64 @@ def test_aiter_moe_shared_experts_enablement_follows_env( assert rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() is expected -def test_aiter_moe_situv2_syncs_aiter_a4w4_env( - monkeypatch: pytest.MonkeyPatch, -): - """SiTUv2 MoE should route to AITER a4w4 and clear legacy a8w4 overrides.""" - from vllm._aiter_ops import rocm_aiter_ops - - _assert_aiter_supported() - - with monkeypatch.context() as mp: - mp.delenv("AITER_SITUV2_A8W4", raising=False) - mp.delenv("AITER_SITUV2_A4W4", raising=False) - mp.setenv("VLLM_ROCM_USE_AITER", "1") - mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1") - mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "1") - _reload_envs() - rocm_aiter_ops.refresh_env_variables() - - import os - - assert os.environ.get("AITER_SITUV2_A4W4") == "1" - assert "AITER_SITUV2_A8W4" not in os.environ - - -def test_aiter_moe_situv2_legacy_a8w4_alias_enables_a4w4( - monkeypatch: pytest.MonkeyPatch, -): - """Deprecated VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4 still enables SiTUv2 a4w4.""" - from vllm._aiter_ops import rocm_aiter_ops - - _assert_aiter_supported() - - with monkeypatch.context() as mp: - mp.delenv("AITER_SITUV2_A8W4", raising=False) - mp.delenv("AITER_SITUV2_A4W4", raising=False) - mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", raising=False) - mp.setenv("VLLM_ROCM_USE_AITER", "1") - mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1") - mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", "1") - _reload_envs() - rocm_aiter_ops.refresh_env_variables() - - import os - - assert os.environ.get("AITER_SITUV2_A4W4") == "1" - assert "AITER_SITUV2_A8W4" not in os.environ - - -def test_aiter_moe_situv2_clears_aiter_env_when_disabled( +@pytest.mark.parametrize( + "value,expected_act,expected_env", + [ + # Unset / auto / legacy "1" all mean the a4w4 default. + (None, "a4w4", {"AITER_SITUV2_A4W4": "1"}), + ("auto", "a4w4", {"AITER_SITUV2_A4W4": "1"}), + ("1", "a4w4", {"AITER_SITUV2_A4W4": "1"}), + ("a4w4", "a4w4", {"AITER_SITUV2_A4W4": "1"}), + ("0", "a16w4", {}), + ("a16w4", "a16w4", {}), + ("A8W4", "a8w4", {"AITER_SITUV2_A8W4": "1"}), + ], +) +def test_aiter_moe_situv2_activation_syncs_aiter_env( + value: str | None, + expected_act: str, + expected_env: dict[str, str], monkeypatch: pytest.MonkeyPatch, ): + """Each SiTUv2 activation choice sets exactly its AITER dispatch env.""" + import os + from vllm._aiter_ops import rocm_aiter_ops _assert_aiter_supported() with monkeypatch.context() as mp: + # Stale AITER overrides must be replaced by the vLLM selection. + mp.setenv("AITER_SITUV2_A8W4", "1") mp.setenv("AITER_SITUV2_A4W4", "1") mp.setenv("VLLM_ROCM_USE_AITER", "1") mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1") - mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", raising=False) - mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "0") + if value is None: + mp.delenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", raising=False) + else: + mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", value) _reload_envs() rocm_aiter_ops.refresh_env_variables() - import os + assert rocm_aiter_ops.get_fused_moe_situv2_activation() == expected_act + assert rocm_aiter_ops.is_fused_moe_situv2_gate_up_interleaved() is ( + expected_act == "a8w4" + ) + assert rocm_aiter_ops.is_fused_moe_situv2_enabled() is (expected_act != "a16w4") + for name in ("AITER_SITUV2_A8W4", "AITER_SITUV2_A4W4"): + assert os.environ.get(name) == expected_env.get(name) - assert "AITER_SITUV2_A4W4" not in os.environ + +def test_aiter_moe_situv2_rejects_unknown_activation( + monkeypatch: pytest.MonkeyPatch, +): + with monkeypatch.context() as mp: + mp.setenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "a8w8") + _reload_envs() + import vllm.envs as envs + + with pytest.raises(ValueError, match="VLLM_ROCM_USE_AITER_MOE_SITUV2"): + _ = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2 @pytest.mark.parametrize("moe_padding", [True, False]) diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index af0213361e..0e89b39818 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -1870,28 +1870,39 @@ def _mhc_delayed_pre_tail( _OPS_REGISTERED = False +_AITER_SITUV2_ACT_ENV = { + "a8w4": "AITER_SITUV2_A8W4", + "a4w4": "AITER_SITUV2_A4W4", +} + + +def _resolve_situv2_activation() -> str: + """VLLM_ROCM_USE_AITER_MOE_SITUV2 -> a4w4 | a8w4 | a16w4. + + auto (default) and the legacy 1 mean a4w4; legacy 0 means a16w4. + """ + value = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2.lower() + if value in ("auto", "1"): + return "a4w4" + if value == "0": + return "a16w4" + return value + + def _sync_aiter_situv2_moe_env() -> None: - """Mirror the SiTUv2 MoE toggle into AITER's a4w4 dispatch env. + """Set the AITER_SITUV2_* env matching VLLM_ROCM_USE_AITER_MOE_SITUV2. - AITER selects afp8 vs afp4 activation kernels via AITER_SITUV2_A8W4 / - AITER_SITUV2_A4W4 (see ROCm/aiter fused_moe.py, A8W4 checked first). - When VLLM_ROCM_USE_AITER_MOE_SITUV2 is enabled we route to a4w4 - (afp4_wfp4_fp4 kernels) and clear any legacy AITER_SITUV2_A8W4 override. - - Requires AITER with ROCm/aiter#4463 (first tagged in v0.1.20): a4w4 - dispatch plus kimik3_a4w4_{un,}tuned_fmoe.csv. Older AITER still runs - afp4 FlyDSL but falls back to heuristic configs, which is not the - tuned a4w4 path this recipe is meant to use. + AITER reads AITER_SITUV2_A8W4 / AITER_SITUV2_A4W4 and defaults to a16w4, + so exactly one is set for a8w4/a4w4 and both are cleared for a16w4. """ import os - import vllm.envs as envs - - if envs.VLLM_ROCM_USE_AITER_MOE_SITUV2: - os.environ["AITER_SITUV2_A4W4"] = "1" - os.environ.pop("AITER_SITUV2_A8W4", None) - else: - os.environ.pop("AITER_SITUV2_A4W4", None) + selected = _AITER_SITUV2_ACT_ENV.get(_resolve_situv2_activation()) + for name in _AITER_SITUV2_ACT_ENV.values(): + if name == selected: + os.environ[name] = "1" + else: + os.environ.pop(name, None) class rocm_aiter_ops: @@ -1917,7 +1928,8 @@ class rocm_aiter_ops: VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: Controls FP4 assembly GEMM. VLLM_ROCM_USE_AITER_TRITON_ROPE: Controls Triton rotary embeddings. VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: Controls shared expert fusion. - VLLM_ROCM_USE_AITER_MOE_SITUV2: Controls SiTUv2 FlyDSL MoE (a4w4). + VLLM_ROCM_USE_AITER_MOE_SITUV2: SiTUv2 FlyDSL MoE activation + dtype (a16w4 | a8w4 | a4w4). VLLM_ROCM_USE_AITER_TRITON_GEMM: Controls Triton unquantized GEMM. Note: @@ -1987,7 +1999,7 @@ class rocm_aiter_ops: # TODO: Consolidate under VLLM_ROCM_USE_AITER_ROPE _TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE _MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS - _MOE_SITUV2 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2 + _MOE_SITUV2 = _resolve_situv2_activation() # TODO: Consolidate under _LINEAR_ENABLED _TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM # Lazily probed: whether aiter.topk_softmax supports the @@ -2016,7 +2028,7 @@ class rocm_aiter_ops: cls._FP4_GEMM_DYNAMIC_QUANT_ASM = envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM cls._TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE cls._MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS - cls._MOE_SITUV2 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2 + cls._MOE_SITUV2 = _resolve_situv2_activation() _sync_aiter_situv2_moe_env() cls._TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM cls._MOE_DISPATCH_POLICY = envs.VLLM_ROCM_AITER_MOE_DISPATCH_POLICY @@ -2132,9 +2144,25 @@ class rocm_aiter_ops: @classmethod @if_aiter_supported def is_fused_moe_situv2_enabled(cls) -> bool: - # _MOE_SITUV2 is a variant of aiter fused moe, so aiter - # fused moe must be enabled as well. - return cls.is_fused_moe_enabled() and cls._MOE_SITUV2 + """True when a low-precision (a8w4/a4w4) SiTUv2 activation is selected.""" + return cls.is_fused_moe_enabled() and cls._MOE_SITUV2 != "a16w4" + + @classmethod + @if_aiter_supported + def get_fused_moe_situv2_activation(cls) -> str: + """SiTUv2 activation dtype name: a16w4, a8w4 or a4w4.""" + return cls._MOE_SITUV2 + + @classmethod + @if_aiter_supported + def is_fused_moe_situv2_gate_up_interleaved(cls) -> bool: + """Whether the SiTUv2 kernels expect gate/up-interleaved w13. + + The a8w4 FlyDSL stage-1 kernels (_gui_) are interleaved; a16w4 and + a4w4 are separated. Weight shuffle (oracle/mxfp4.py) and gate_mode + (experts/rocm_aiter_moe.py) must both use this. + """ + return cls._MOE_SITUV2 == "a8w4" @classmethod @if_aiter_supported diff --git a/vllm/envs.py b/vllm/envs.py index 78f99ea0ac..b0d72a71db 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -141,7 +141,7 @@ if TYPE_CHECKING: VLLM_ROCM_USE_AITER_LINEAR_HIPBMM: bool = False VLLM_ROCM_USE_AITER_MOE: bool = True VLLM_ROCM_AITER_MOE_DISPATCH_POLICY: int = 0 - VLLM_ROCM_USE_AITER_MOE_SITUV2: bool = False + VLLM_ROCM_USE_AITER_MOE_SITUV2: Literal["auto", "a4w4", "a8w4", "a16w4"] = "auto" VLLM_ROCM_USE_AITER_RMSNORM: bool = True VLLM_ROCM_USE_AITER_MLA: bool = True VLLM_ROCM_AITER_MLA_ASM_PADDING: Literal["auto", "gluon", "asm"] = "auto" @@ -1280,18 +1280,13 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_ROCM_USE_AITER_MOE": lambda: ( os.getenv("VLLM_ROCM_USE_AITER_MOE", "True").lower() in ("true", "1") ), - # Route K3 SiTU MXFP4 MoE through the FlyDSL SiTUv2 path (a4w4 fp4 - # activations, separated gate/up layout) instead of default a16w4. vLLM - # sets AITER_SITUV2_A4W4 at init when this flag is on and clears any - # legacy AITER_SITUV2_A8W4 override (AITER checks A8W4 first). - # VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4 is a deprecated alias for existing - # recipes; it does not select a8w4 kernels. - # Needs AITER >= v0.1.20 (ROCm/aiter#4463) for the a4w4 dispatch flag - # and tuned kimik3_a4w4_*_fmoe.csv rows; otherwise FlyDSL uses heuristics. - "VLLM_ROCM_USE_AITER_MOE_SITUV2": lambda: ( - os.getenv("VLLM_ROCM_USE_AITER_MOE_SITUV2", "0").lower() in ("true", "1") - or os.getenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", "0").lower() - in ("true", "1") + # Activation dtype for the Kimi-K3 SiTU MXFP4 MoE (AITER FlyDSL SiTUv2): + # auto (= a4w4), a4w4, a8w4 or a16w4. Legacy 1/0 mean a4w4/a16w4. + "VLLM_ROCM_USE_AITER_MOE_SITUV2": env_with_choices( + "VLLM_ROCM_USE_AITER_MOE_SITUV2", + "auto", + ["auto", "a4w4", "a8w4", "a16w4", "0", "1"], + case_sensitive=False, ), # MoE sorting dispatch policy for AITER fused MoE kernels. # 0 = auto (default): single-pass for small batches, multi-pass diff --git a/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py b/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py index 62df395f69..4f2f760ad3 100644 --- a/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py @@ -385,10 +385,11 @@ def rocm_aiter_fused_experts( gate_mode = "" if activation == MoEActivation.SITU: - # SiTUv2 flydsl (VLLM_ROCM_USE_AITER_MOE_SITUV2=1) uses a4w4 - # fp4 activations with separated gate/up weights (AITER #4463); - # default a16w4 SiTU also stays separated. - gate_mode = GateMode.SEPARATED.value + gate_mode = ( + GateMode.INTERLEAVE.value + if rocm_aiter_ops.is_fused_moe_situv2_gate_up_interleaved() + else GateMode.SEPARATED.value + ) elif quant_config.use_mxfp4_w4a16: gate_mode = GateMode.INTERLEAVE.value elif activation_interleave is not None: diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py index 5450f73f85..56ba3981b7 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py @@ -1596,8 +1596,7 @@ def convert_weight_to_mxfp4_moe_kernel_format( fp4_dtype = torch.float4_e2m1fn_x2 e8m0_dtype = torch.float8_e8m0fnu - # SiTUv2 flydsl uses separated gate/up layout (a4w4). - guinterleave = False + guinterleave = rocm_aiter_ops.is_fused_moe_situv2_gate_up_interleaved() w13 = rocm_aiter_ops.shuffle_weight_a16w4( w13_weight.data.view(fp4_dtype), 16, guinterleave )