diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index 30bfcb80b0..990016e161 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -1530,6 +1530,7 @@ class rocm_aiter_ops: VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: Controls FP4 assembly GEMM. VLLM_ROCM_USE_AITER_TRITON_ROPE: Controls Triton rotary embeddings. VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: Controls shared expert fusion. + VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4: Controls a8w4 SiTU fused MoE variant. VLLM_ROCM_USE_AITER_TRITON_GEMM: Controls Triton unquantized GEMM. Note: @@ -1598,6 +1599,7 @@ class rocm_aiter_ops: # TODO: Consolidate under VLLM_ROCM_USE_AITER_ROPE _TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE _MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS + _MOE_SITUV2_A8W4 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4 # TODO: Consolidate under _LINEAR_ENABLED _TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM # Lazily probed: whether aiter.topk_softmax supports the @@ -1627,6 +1629,7 @@ class rocm_aiter_ops: cls._FP4_GEMM_DYNAMIC_QUANT_ASM = envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM cls._TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE cls._MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS + cls._MOE_SITUV2_A8W4 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4 cls._TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM @staticmethod @@ -1719,6 +1722,13 @@ class rocm_aiter_ops: def is_fusion_moe_shared_experts_enabled(cls) -> bool: return cls.is_fused_moe_enabled() and cls._MOE_SHARED_EXPERTS_ENABLED + @classmethod + @if_aiter_supported + def is_fused_moe_situv2_a8w4_enabled(cls) -> bool: + # _MOE_SITUV2_A8W4 is a variant of aiter fused moe, so aiter + # fused moe must be enabled as well. + return cls.is_fused_moe_enabled() and cls._MOE_SITUV2_A8W4 + @classmethod @if_aiter_supported def topk_softmax_supports_fused_sigmoid(cls) -> bool: diff --git a/vllm/envs.py b/vllm/envs.py index da45b92703..0b16f78fcd 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -130,7 +130,7 @@ if TYPE_CHECKING: VLLM_ROCM_USE_AITER_LINEAR_HIPBMM: bool = False VLLM_ROCM_USE_AITER_MOE: bool = True VLLM_ROCM_AITER_MOE_DISPATCH_POLICY: int = 0 - AITER_SITUV2_A8W4: bool = False + VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4: bool = False VLLM_ROCM_USE_AITER_RMSNORM: bool = True VLLM_ROCM_USE_AITER_MLA: bool = True VLLM_ROCM_USE_AITER_MHA: bool = True @@ -1211,9 +1211,10 @@ environment_variables: dict[str, Callable[[], Any]] = { ), # Route K3 SiTU MXFP4 MoE through the a8w4 (fp8 activation) gate/up- # interleaved flydsl kernels instead of the default a16w4 separated path. - # Shared with the AITER runtime, which reads the same env var directly. - "AITER_SITUV2_A8W4": lambda: ( - os.getenv("AITER_SITUV2_A8W4", "0").lower() in ("true", "1") + # This is the only flag users need: vLLM picks the kernels by passing + # gate_mode to AITER and sets the AITER-side workaround env at init. + "VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4": lambda: ( + os.getenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", "0").lower() in ("true", "1") ), # MoE sorting dispatch policy for AITER fused MoE kernels. # 0 = auto (default): single-pass for small batches, multi-pass diff --git a/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py b/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py index 096e729480..c737109f47 100644 --- a/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py @@ -5,7 +5,6 @@ from functools import lru_cache import torch -import vllm.envs as envs import vllm.model_executor.layers.fused_moe.modular_kernel as mk from vllm._aiter_ops import rocm_aiter_ops from vllm.model_executor.layers.fused_moe.activation import MoEActivation @@ -376,11 +375,12 @@ def rocm_aiter_fused_experts( gate_mode = "" if activation == MoEActivation.SITU: - # a8w4 (AITER_SITUV2_A8W4=1) uses the gate/up-interleaved (_gui_) - # fp8 flydsl kernels; default a16w4 SiTU stays separated. + # a8w4 (VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4=1) uses the gate/up- + # interleaved (_gui_) fp8 flydsl kernels; default a16w4 SiTU stays + # separated. gate_mode = ( GateMode.INTERLEAVE.value - if envs.AITER_SITUV2_A8W4 + if rocm_aiter_ops.is_fused_moe_situv2_a8w4_enabled() else GateMode.SEPARATED.value ) elif quant_config.use_mxfp4_w4a16: diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py index 1e1d55ac9c..e8371e4ab4 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py @@ -14,6 +14,7 @@ from vllm.model_executor.layers.fused_moe import ( FusedMoEConfig, RoutedExperts, ) +from vllm.model_executor.layers.fused_moe.activation import MoEActivation from vllm.model_executor.layers.fused_moe.all2all_utils import ( maybe_make_prepare_finalize, ) @@ -634,10 +635,20 @@ def select_deepseek_v4_mxfp4_moe_backend( def mxfp4_round_up_hidden_size_and_intermediate_size( - backend: Mxfp4MoeBackend, hidden_size: int, intermediate_size: int + backend: Mxfp4MoeBackend, + hidden_size: int, + intermediate_size: int, + activation: MoEActivation | None = None, ) -> tuple[int, int]: """Round up hidden_size and intermediate_size based on backend requirements.""" - if backend == Mxfp4MoeBackend.EMULATION: + if backend == Mxfp4MoeBackend.AITER_MXFP4_BF16 and activation == MoEActivation.SITU: + # K3's AITER A16W4 SiTU kernel handles K3's native intermediate size + # (moe_intermediate 3072; e.g. 384/partition at TP8). Align to 128 (a + # no-op for K3's shapes) rather than the generic ROCm 256 round-up, + # which would inflate weights and OOM. + intermediate_size = round_up(intermediate_size, 128) + hidden_size = round_up(hidden_size, 128) + elif backend == Mxfp4MoeBackend.EMULATION: # Emulation has no kernel tile; it only needs OCP MX block alignment so the # per-block scale buffers (`dim // OCP_MX_BLOCK_SIZE`) aren't floor-truncated # by a non-block-aligned TP/DP shard (e.g. 2880 // 4 = 720). diff --git a/vllm/model_executor/layers/quantization/mxfp4.py b/vllm/model_executor/layers/quantization/mxfp4.py index 1ee1ad75fd..266508a005 100644 --- a/vllm/model_executor/layers/quantization/mxfp4.py +++ b/vllm/model_executor/layers/quantization/mxfp4.py @@ -1,9 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import os + import torch -import vllm.envs as envs from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( @@ -515,6 +516,14 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): self.mxfp4_backend = Mxfp4MoeBackend.AITER_MXFP4_BF16 self.experts_cls = backend_to_kernel_cls(self.mxfp4_backend)[0] logger.info_once("Using AITER_MXFP4_BF16 for Kimi-K3 SiTU MXFP4 MoE.") + from vllm._aiter_ops import rocm_aiter_ops + + if rocm_aiter_ops.is_fused_moe_situv2_a8w4_enabled(): + # AITER keeps bf16 activations below this token count, which + # would not match the fp8 a8w4 kernels the interleaved SiTU + # path is tuned for. The a16w4 path never reads it. + # TODO: Remove once AITER takes this as a kernel argument. + os.environ["AITER_BF16_FP8_MOE_BOUND"] = "0" else: self.mxfp4_backend, self.experts_cls = select_deepseek_v4_mxfp4_moe_backend( moe @@ -560,13 +569,11 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): act_dtype=act_dtype, moe_parallel_config=moe_parallel_config, ) - if self.is_k3_situ_aiter: - # K3's AITER A16W4 kernel handles K3's native intermediate size - # (moe_intermediate 3072; e.g. 384/partition at TP8); the generic - # 256 round-up would inflate weights and OOM. - return hidden_size, intermediate_size_per_partition return mxfp4_round_up_hidden_size_and_intermediate_size( - self.mxfp4_backend, hidden_size, intermediate_size_per_partition + self.mxfp4_backend, + hidden_size, + intermediate_size_per_partition, + activation=self.moe.activation, ) def create_weights( @@ -716,19 +723,24 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): ) # Convert weights to kernel format - w13, w2, w13_scale, w2_scale, w13_bias, w2_bias = ( - convert_weight_to_mxfp4_moe_kernel_format( - mxfp4_backend=self.mxfp4_backend, - layer=layer, - w13_weight=w13, - w2_weight=w2, - w13_weight_scale=w13_scale, - w2_weight_scale=w2_scale, - w13_bias=w13_bias, - w2_bias=w2_bias, - _cache_permute_indices=self._cache_permute_indices, + if self.is_k3_situ_aiter: + w13, w2, w13_scale, w2_scale = ( + self._convert_k3_situ_weight_to_kernel_format(layer) + ) + else: + w13, w2, w13_scale, w2_scale, w13_bias, w2_bias = ( + convert_weight_to_mxfp4_moe_kernel_format( + mxfp4_backend=self.mxfp4_backend, + layer=layer, + w13_weight=w13, + w2_weight=w2, + w13_weight_scale=w13_scale, + w2_weight_scale=w2_scale, + w13_bias=w13_bias, + w2_bias=w2_bias, + _cache_permute_indices=self._cache_permute_indices, + ) ) - ) # For TRITON backends, weights are wrapped tensors from triton_kernels # that don't support .detach(). Manually assign parameters. @@ -774,7 +786,9 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): layer=layer, ) - def _setup_kernel_k3_situ(self, layer: RoutedExperts) -> None: + def _convert_k3_situ_weight_to_kernel_format( + self, layer: RoutedExperts + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: # K3's AITER A16W4 kernel wants the separated ([gate_all, up_all]) # stage-1 layout, unlike the interleaved gpt-oss/DeepSeek path in # convert_weight_to_mxfp4_moe_kernel_format. Preshuffle once here. @@ -786,10 +800,10 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): e8m0_dtype = torch.float8_e8m0fnu num_experts = layer.w13_weight.shape[0] - # a8w4 (AITER_SITUV2_A8W4=1) uses the gate/up-interleaved (_gui_) fp8 - # flydsl kernels, which need w13 weight+scale in interleave layout. - # Default a16w4 keeps the separated layout. - guinterleave = envs.AITER_SITUV2_A8W4 + # a8w4 (VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4=1) uses the gate/up- + # interleaved (_gui_) fp8 flydsl kernels, which need w13 weight+scale + # in interleave layout. Default a16w4 keeps the separated layout. + guinterleave = rocm_aiter_ops.is_fused_moe_situv2_a8w4_enabled() w13 = rocm_aiter_ops.shuffle_weight_a16w4( layer.w13_weight.data.view(fp4_dtype), 16, guinterleave ) @@ -802,30 +816,9 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): w13_scale_raw.view(-1, w13_scale_raw.shape[-1]), num_experts, guinterleave ) w2_scale = e8m0_shuffle(w2_scale_raw.view(-1, w2_scale_raw.shape[-1])) - - replace_parameter(layer, "w13_weight", w13) - replace_parameter(layer, "w2_weight", w2) - replace_parameter(layer, "w13_weight_scale", w13_scale) - replace_parameter(layer, "w2_weight_scale", w2_scale) - layer.w13_weight.is_shuffled = True - layer.w2_weight.is_shuffled = True - - self.moe_quant_config = self.get_fused_moe_quant_config(layer) - if self.moe_quant_config is not None and self.experts_cls is not None: - self.moe_kernel = make_mxfp4_moe_kernel( - moe_quant_config=self.moe_quant_config, - moe_config=self.moe, - mxfp4_backend=self.mxfp4_backend, - experts_cls=self.experts_cls, - routing_tables=layer._expert_routing_tables(), - layer=layer, - ) + return w13, w2, w13_scale, w2_scale def process_weights_after_loading(self, layer): - if self.is_k3_situ_aiter: - self._setup_kernel_k3_situ(layer) - return - w13 = layer.w13_weight w2 = layer.w2_weight w13_scale = layer.w13_weight_scale