[ROCm][Kimi-K3] aiter moe environment variable cleanup (#50582)
Signed-off-by: Hongxia Yang <hongxia.yang@amd.com>
This commit is contained in:
@@ -1530,6 +1530,7 @@ class rocm_aiter_ops:
|
||||
VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: Controls FP4 assembly GEMM.
|
||||
VLLM_ROCM_USE_AITER_TRITON_ROPE: Controls Triton rotary embeddings.
|
||||
VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: Controls shared expert fusion.
|
||||
VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4: Controls a8w4 SiTU fused MoE variant.
|
||||
VLLM_ROCM_USE_AITER_TRITON_GEMM: Controls Triton unquantized GEMM.
|
||||
|
||||
Note:
|
||||
@@ -1598,6 +1599,7 @@ class rocm_aiter_ops:
|
||||
# TODO: Consolidate under VLLM_ROCM_USE_AITER_ROPE
|
||||
_TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE
|
||||
_MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
|
||||
_MOE_SITUV2_A8W4 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4
|
||||
# TODO: Consolidate under _LINEAR_ENABLED
|
||||
_TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM
|
||||
# Lazily probed: whether aiter.topk_softmax supports the
|
||||
@@ -1627,6 +1629,7 @@ class rocm_aiter_ops:
|
||||
cls._FP4_GEMM_DYNAMIC_QUANT_ASM = envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM
|
||||
cls._TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE
|
||||
cls._MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
|
||||
cls._MOE_SITUV2_A8W4 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4
|
||||
cls._TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM
|
||||
|
||||
@staticmethod
|
||||
@@ -1719,6 +1722,13 @@ class rocm_aiter_ops:
|
||||
def is_fusion_moe_shared_experts_enabled(cls) -> bool:
|
||||
return cls.is_fused_moe_enabled() and cls._MOE_SHARED_EXPERTS_ENABLED
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
def is_fused_moe_situv2_a8w4_enabled(cls) -> bool:
|
||||
# _MOE_SITUV2_A8W4 is a variant of aiter fused moe, so aiter
|
||||
# fused moe must be enabled as well.
|
||||
return cls.is_fused_moe_enabled() and cls._MOE_SITUV2_A8W4
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
def topk_softmax_supports_fused_sigmoid(cls) -> bool:
|
||||
|
||||
+5
-4
@@ -130,7 +130,7 @@ if TYPE_CHECKING:
|
||||
VLLM_ROCM_USE_AITER_LINEAR_HIPBMM: bool = False
|
||||
VLLM_ROCM_USE_AITER_MOE: bool = True
|
||||
VLLM_ROCM_AITER_MOE_DISPATCH_POLICY: int = 0
|
||||
AITER_SITUV2_A8W4: bool = False
|
||||
VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4: bool = False
|
||||
VLLM_ROCM_USE_AITER_RMSNORM: bool = True
|
||||
VLLM_ROCM_USE_AITER_MLA: bool = True
|
||||
VLLM_ROCM_USE_AITER_MHA: bool = True
|
||||
@@ -1211,9 +1211,10 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
),
|
||||
# Route K3 SiTU MXFP4 MoE through the a8w4 (fp8 activation) gate/up-
|
||||
# interleaved flydsl kernels instead of the default a16w4 separated path.
|
||||
# Shared with the AITER runtime, which reads the same env var directly.
|
||||
"AITER_SITUV2_A8W4": lambda: (
|
||||
os.getenv("AITER_SITUV2_A8W4", "0").lower() in ("true", "1")
|
||||
# This is the only flag users need: vLLM picks the kernels by passing
|
||||
# gate_mode to AITER and sets the AITER-side workaround env at init.
|
||||
"VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4": lambda: (
|
||||
os.getenv("VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4", "0").lower() in ("true", "1")
|
||||
),
|
||||
# MoE sorting dispatch policy for AITER fused MoE kernels.
|
||||
# 0 = auto (default): single-pass for small batches, multi-pass
|
||||
|
||||
@@ -5,7 +5,6 @@ from functools import lru_cache
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
@@ -376,11 +375,12 @@ def rocm_aiter_fused_experts(
|
||||
|
||||
gate_mode = ""
|
||||
if activation == MoEActivation.SITU:
|
||||
# a8w4 (AITER_SITUV2_A8W4=1) uses the gate/up-interleaved (_gui_)
|
||||
# fp8 flydsl kernels; default a16w4 SiTU stays separated.
|
||||
# a8w4 (VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4=1) uses the gate/up-
|
||||
# interleaved (_gui_) fp8 flydsl kernels; default a16w4 SiTU stays
|
||||
# separated.
|
||||
gate_mode = (
|
||||
GateMode.INTERLEAVE.value
|
||||
if envs.AITER_SITUV2_A8W4
|
||||
if rocm_aiter_ops.is_fused_moe_situv2_a8w4_enabled()
|
||||
else GateMode.SEPARATED.value
|
||||
)
|
||||
elif quant_config.use_mxfp4_w4a16:
|
||||
|
||||
@@ -14,6 +14,7 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEConfig,
|
||||
RoutedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
maybe_make_prepare_finalize,
|
||||
)
|
||||
@@ -634,10 +635,20 @@ def select_deepseek_v4_mxfp4_moe_backend(
|
||||
|
||||
|
||||
def mxfp4_round_up_hidden_size_and_intermediate_size(
|
||||
backend: Mxfp4MoeBackend, hidden_size: int, intermediate_size: int
|
||||
backend: Mxfp4MoeBackend,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
activation: MoEActivation | None = None,
|
||||
) -> tuple[int, int]:
|
||||
"""Round up hidden_size and intermediate_size based on backend requirements."""
|
||||
if backend == Mxfp4MoeBackend.EMULATION:
|
||||
if backend == Mxfp4MoeBackend.AITER_MXFP4_BF16 and activation == MoEActivation.SITU:
|
||||
# K3's AITER A16W4 SiTU kernel handles K3's native intermediate size
|
||||
# (moe_intermediate 3072; e.g. 384/partition at TP8). Align to 128 (a
|
||||
# no-op for K3's shapes) rather than the generic ROCm 256 round-up,
|
||||
# which would inflate weights and OOM.
|
||||
intermediate_size = round_up(intermediate_size, 128)
|
||||
hidden_size = round_up(hidden_size, 128)
|
||||
elif backend == Mxfp4MoeBackend.EMULATION:
|
||||
# Emulation has no kernel tile; it only needs OCP MX block alignment so the
|
||||
# per-block scale buffers (`dim // OCP_MX_BLOCK_SIZE`) aren't floor-truncated
|
||||
# by a non-block-aligned TP/DP shard (e.g. 2880 // 4 = 720).
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
@@ -515,6 +516,14 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
self.mxfp4_backend = Mxfp4MoeBackend.AITER_MXFP4_BF16
|
||||
self.experts_cls = backend_to_kernel_cls(self.mxfp4_backend)[0]
|
||||
logger.info_once("Using AITER_MXFP4_BF16 for Kimi-K3 SiTU MXFP4 MoE.")
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
if rocm_aiter_ops.is_fused_moe_situv2_a8w4_enabled():
|
||||
# AITER keeps bf16 activations below this token count, which
|
||||
# would not match the fp8 a8w4 kernels the interleaved SiTU
|
||||
# path is tuned for. The a16w4 path never reads it.
|
||||
# TODO: Remove once AITER takes this as a kernel argument.
|
||||
os.environ["AITER_BF16_FP8_MOE_BOUND"] = "0"
|
||||
else:
|
||||
self.mxfp4_backend, self.experts_cls = select_deepseek_v4_mxfp4_moe_backend(
|
||||
moe
|
||||
@@ -560,13 +569,11 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
act_dtype=act_dtype,
|
||||
moe_parallel_config=moe_parallel_config,
|
||||
)
|
||||
if self.is_k3_situ_aiter:
|
||||
# K3's AITER A16W4 kernel handles K3's native intermediate size
|
||||
# (moe_intermediate 3072; e.g. 384/partition at TP8); the generic
|
||||
# 256 round-up would inflate weights and OOM.
|
||||
return hidden_size, intermediate_size_per_partition
|
||||
return mxfp4_round_up_hidden_size_and_intermediate_size(
|
||||
self.mxfp4_backend, hidden_size, intermediate_size_per_partition
|
||||
self.mxfp4_backend,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition,
|
||||
activation=self.moe.activation,
|
||||
)
|
||||
|
||||
def create_weights(
|
||||
@@ -716,19 +723,24 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
|
||||
# Convert weights to kernel format
|
||||
w13, w2, w13_scale, w2_scale, w13_bias, w2_bias = (
|
||||
convert_weight_to_mxfp4_moe_kernel_format(
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
layer=layer,
|
||||
w13_weight=w13,
|
||||
w2_weight=w2,
|
||||
w13_weight_scale=w13_scale,
|
||||
w2_weight_scale=w2_scale,
|
||||
w13_bias=w13_bias,
|
||||
w2_bias=w2_bias,
|
||||
_cache_permute_indices=self._cache_permute_indices,
|
||||
if self.is_k3_situ_aiter:
|
||||
w13, w2, w13_scale, w2_scale = (
|
||||
self._convert_k3_situ_weight_to_kernel_format(layer)
|
||||
)
|
||||
else:
|
||||
w13, w2, w13_scale, w2_scale, w13_bias, w2_bias = (
|
||||
convert_weight_to_mxfp4_moe_kernel_format(
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
layer=layer,
|
||||
w13_weight=w13,
|
||||
w2_weight=w2,
|
||||
w13_weight_scale=w13_scale,
|
||||
w2_weight_scale=w2_scale,
|
||||
w13_bias=w13_bias,
|
||||
w2_bias=w2_bias,
|
||||
_cache_permute_indices=self._cache_permute_indices,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
# For TRITON backends, weights are wrapped tensors from triton_kernels
|
||||
# that don't support .detach(). Manually assign parameters.
|
||||
@@ -774,7 +786,9 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def _setup_kernel_k3_situ(self, layer: RoutedExperts) -> None:
|
||||
def _convert_k3_situ_weight_to_kernel_format(
|
||||
self, layer: RoutedExperts
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
# K3's AITER A16W4 kernel wants the separated ([gate_all, up_all])
|
||||
# stage-1 layout, unlike the interleaved gpt-oss/DeepSeek path in
|
||||
# convert_weight_to_mxfp4_moe_kernel_format. Preshuffle once here.
|
||||
@@ -786,10 +800,10 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
e8m0_dtype = torch.float8_e8m0fnu
|
||||
num_experts = layer.w13_weight.shape[0]
|
||||
|
||||
# a8w4 (AITER_SITUV2_A8W4=1) uses the gate/up-interleaved (_gui_) fp8
|
||||
# flydsl kernels, which need w13 weight+scale in interleave layout.
|
||||
# Default a16w4 keeps the separated layout.
|
||||
guinterleave = envs.AITER_SITUV2_A8W4
|
||||
# a8w4 (VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4=1) uses the gate/up-
|
||||
# interleaved (_gui_) fp8 flydsl kernels, which need w13 weight+scale
|
||||
# in interleave layout. Default a16w4 keeps the separated layout.
|
||||
guinterleave = rocm_aiter_ops.is_fused_moe_situv2_a8w4_enabled()
|
||||
w13 = rocm_aiter_ops.shuffle_weight_a16w4(
|
||||
layer.w13_weight.data.view(fp4_dtype), 16, guinterleave
|
||||
)
|
||||
@@ -802,30 +816,9 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
w13_scale_raw.view(-1, w13_scale_raw.shape[-1]), num_experts, guinterleave
|
||||
)
|
||||
w2_scale = e8m0_shuffle(w2_scale_raw.view(-1, w2_scale_raw.shape[-1]))
|
||||
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
replace_parameter(layer, "w2_weight", w2)
|
||||
replace_parameter(layer, "w13_weight_scale", w13_scale)
|
||||
replace_parameter(layer, "w2_weight_scale", w2_scale)
|
||||
layer.w13_weight.is_shuffled = True
|
||||
layer.w2_weight.is_shuffled = True
|
||||
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
if self.moe_quant_config is not None and self.experts_cls is not None:
|
||||
self.moe_kernel = make_mxfp4_moe_kernel(
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
return w13, w2, w13_scale, w2_scale
|
||||
|
||||
def process_weights_after_loading(self, layer):
|
||||
if self.is_k3_situ_aiter:
|
||||
self._setup_kernel_k3_situ(layer)
|
||||
return
|
||||
|
||||
w13 = layer.w13_weight
|
||||
w2 = layer.w2_weight
|
||||
w13_scale = layer.w13_weight_scale
|
||||
|
||||
Reference in New Issue
Block a user