[ROCm][Kimi-K3] aiter moe environment variable cleanup (#50582)

Signed-off-by: Hongxia Yang <hongxia.yang@amd.com>
This commit is contained in:
Hongxia Yang
2026-08-03 18:54:04 +00:00
committed by GitHub
parent 952694e384
commit 8f50685c48
5 changed files with 71 additions and 56 deletions
@@ -1,9 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
import torch
import vllm.envs as envs
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
@@ -515,6 +516,14 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
self.mxfp4_backend = Mxfp4MoeBackend.AITER_MXFP4_BF16
self.experts_cls = backend_to_kernel_cls(self.mxfp4_backend)[0]
logger.info_once("Using AITER_MXFP4_BF16 for Kimi-K3 SiTU MXFP4 MoE.")
from vllm._aiter_ops import rocm_aiter_ops
if rocm_aiter_ops.is_fused_moe_situv2_a8w4_enabled():
# AITER keeps bf16 activations below this token count, which
# would not match the fp8 a8w4 kernels the interleaved SiTU
# path is tuned for. The a16w4 path never reads it.
# TODO: Remove once AITER takes this as a kernel argument.
os.environ["AITER_BF16_FP8_MOE_BOUND"] = "0"
else:
self.mxfp4_backend, self.experts_cls = select_deepseek_v4_mxfp4_moe_backend(
moe
@@ -560,13 +569,11 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
act_dtype=act_dtype,
moe_parallel_config=moe_parallel_config,
)
if self.is_k3_situ_aiter:
# K3's AITER A16W4 kernel handles K3's native intermediate size
# (moe_intermediate 3072; e.g. 384/partition at TP8); the generic
# 256 round-up would inflate weights and OOM.
return hidden_size, intermediate_size_per_partition
return mxfp4_round_up_hidden_size_and_intermediate_size(
self.mxfp4_backend, hidden_size, intermediate_size_per_partition
self.mxfp4_backend,
hidden_size,
intermediate_size_per_partition,
activation=self.moe.activation,
)
def create_weights(
@@ -716,19 +723,24 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
)
# Convert weights to kernel format
w13, w2, w13_scale, w2_scale, w13_bias, w2_bias = (
convert_weight_to_mxfp4_moe_kernel_format(
mxfp4_backend=self.mxfp4_backend,
layer=layer,
w13_weight=w13,
w2_weight=w2,
w13_weight_scale=w13_scale,
w2_weight_scale=w2_scale,
w13_bias=w13_bias,
w2_bias=w2_bias,
_cache_permute_indices=self._cache_permute_indices,
if self.is_k3_situ_aiter:
w13, w2, w13_scale, w2_scale = (
self._convert_k3_situ_weight_to_kernel_format(layer)
)
else:
w13, w2, w13_scale, w2_scale, w13_bias, w2_bias = (
convert_weight_to_mxfp4_moe_kernel_format(
mxfp4_backend=self.mxfp4_backend,
layer=layer,
w13_weight=w13,
w2_weight=w2,
w13_weight_scale=w13_scale,
w2_weight_scale=w2_scale,
w13_bias=w13_bias,
w2_bias=w2_bias,
_cache_permute_indices=self._cache_permute_indices,
)
)
)
# For TRITON backends, weights are wrapped tensors from triton_kernels
# that don't support .detach(). Manually assign parameters.
@@ -774,7 +786,9 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
layer=layer,
)
def _setup_kernel_k3_situ(self, layer: RoutedExperts) -> None:
def _convert_k3_situ_weight_to_kernel_format(
self, layer: RoutedExperts
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
# K3's AITER A16W4 kernel wants the separated ([gate_all, up_all])
# stage-1 layout, unlike the interleaved gpt-oss/DeepSeek path in
# convert_weight_to_mxfp4_moe_kernel_format. Preshuffle once here.
@@ -786,10 +800,10 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
e8m0_dtype = torch.float8_e8m0fnu
num_experts = layer.w13_weight.shape[0]
# a8w4 (AITER_SITUV2_A8W4=1) uses the gate/up-interleaved (_gui_) fp8
# flydsl kernels, which need w13 weight+scale in interleave layout.
# Default a16w4 keeps the separated layout.
guinterleave = envs.AITER_SITUV2_A8W4
# a8w4 (VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4=1) uses the gate/up-
# interleaved (_gui_) fp8 flydsl kernels, which need w13 weight+scale
# in interleave layout. Default a16w4 keeps the separated layout.
guinterleave = rocm_aiter_ops.is_fused_moe_situv2_a8w4_enabled()
w13 = rocm_aiter_ops.shuffle_weight_a16w4(
layer.w13_weight.data.view(fp4_dtype), 16, guinterleave
)
@@ -802,30 +816,9 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
w13_scale_raw.view(-1, w13_scale_raw.shape[-1]), num_experts, guinterleave
)
w2_scale = e8m0_shuffle(w2_scale_raw.view(-1, w2_scale_raw.shape[-1]))
replace_parameter(layer, "w13_weight", w13)
replace_parameter(layer, "w2_weight", w2)
replace_parameter(layer, "w13_weight_scale", w13_scale)
replace_parameter(layer, "w2_weight_scale", w2_scale)
layer.w13_weight.is_shuffled = True
layer.w2_weight.is_shuffled = True
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
if self.moe_quant_config is not None and self.experts_cls is not None:
self.moe_kernel = make_mxfp4_moe_kernel(
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
mxfp4_backend=self.mxfp4_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
return w13, w2, w13_scale, w2_scale
def process_weights_after_loading(self, layer):
if self.is_k3_situ_aiter:
self._setup_kernel_k3_situ(layer)
return
w13 = layer.w13_weight
w2 = layer.w2_weight
w13_scale = layer.w13_weight_scale