[Refactor] Abstract DeepGemm MegaMOE backend (#58053)

Signed-off-by: Hui Shi <kalinshi@tencent.com>
Co-authored-by: LuneExplorer <liyihan23@mails.tsinghua.edu.cn>
Co-authored-by: ArmageddonKnight <bojian@cs.toronto.edu>
This commit is contained in:
Hui Shi
2026-10-08 05:05:39 +00:00
committed by GitHub
co-authored by LuneExplorer ArmageddonKnight
parent 4ac0179f8a
commit 81198e97ba
5 changed files with 683 additions and 268 deletions
+22 -31
View File
@@ -365,27 +365,16 @@ class KimiK3MegaMoEExperts(DeepseekV4MegaMoEExperts):
return
self._check_runtime_supported()
from vllm.utils.deep_gemm import _import_deep_gemm
deep_gemm = _import_deep_gemm()
w13_scale = deep_gemm.transform_sf_into_required_layout(
self._ue8m0_uint8_to_float(self.w13_weight_scale.data).contiguous(),
2 * self.intermediate_size,
self.hidden_size,
(1, 32),
self.num_local_experts,
)
w2_scale = deep_gemm.transform_sf_into_required_layout(
self._ue8m0_uint8_to_float(self.w2_weight_scale.data).contiguous(),
self.hidden_size,
self.intermediate_size,
(1, 32),
self.num_local_experts,
)
backend = self._ensure_backend()
self._transformed_l1_weights, self._transformed_l2_weights = (
deep_gemm.transform_weights_for_mega_moe(
(self.w13_weight.data.view(torch.int8).contiguous(), w13_scale),
(self.w2_weight.data.view(torch.int8).contiguous(), w2_scale),
backend.transform_weights(
w13_weight=self.w13_weight.data,
w13_weight_scale=self.w13_weight_scale.data,
w2_weight=self.w2_weight.data,
w2_weight_scale=self.w2_weight_scale.data,
num_local_experts=self.num_local_experts,
hidden_size=self.hidden_size,
intermediate_size=self.intermediate_size,
activation=self.activation,
)
)
@@ -407,6 +396,7 @@ class KimiK3MegaMoEExperts(DeepseekV4MegaMoEExperts):
from vllm.utils.deep_gemm import _import_deep_gemm
deep_gemm = _import_deep_gemm()
backend = self._ensure_backend()
group = get_ep_group().device_group
device = torch.accelerator.current_device_index()
key = (
@@ -418,6 +408,7 @@ class KimiK3MegaMoEExperts(DeepseekV4MegaMoEExperts):
self.hidden_size,
self.intermediate_size,
self.activation,
backend.mma_type,
)
symm_buffer = self._kimi_symm_buffer_cache.get(key)
if symm_buffer is None:
@@ -429,6 +420,7 @@ class KimiK3MegaMoEExperts(DeepseekV4MegaMoEExperts):
self.hidden_size,
self.intermediate_size,
activation=self.activation,
mma_type=backend.mma_type,
)
self._kimi_symm_buffer_cache[key] = symm_buffer
return symm_buffer
@@ -449,10 +441,6 @@ class KimiK3MegaMoEExperts(DeepseekV4MegaMoEExperts):
f"but its symmetric buffer supports {self.max_num_tokens}."
)
y = torch.empty_like(hidden_states, dtype=torch.bfloat16)
from vllm.utils.deep_gemm import _import_deep_gemm
deep_gemm = _import_deep_gemm()
symm_buffer = self.get_symm_buffer()
num_tokens = hidden_states.shape[0]
is_padding = None
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
@@ -483,6 +471,8 @@ class KimiK3MegaMoEExperts(DeepseekV4MegaMoEExperts):
else None,
)
backend = self._ensure_backend()
symm_buffer = self.get_symm_buffer()
prepare_megamoe_inputs(
hidden_states,
topk_weights,
@@ -492,22 +482,23 @@ class KimiK3MegaMoEExperts(DeepseekV4MegaMoEExperts):
symm_buffer.topk_idx[:num_tokens],
symm_buffer.topk_weights[:num_tokens],
is_padding=is_padding,
hidden_quant=backend.hidden_quant,
)
self.finalize_weights()
assert self._transformed_l1_weights is not None
assert self._transformed_l2_weights is not None
deep_gemm.fp8_fp4_mega_moe(
y,
self._transformed_l1_weights,
self._transformed_l2_weights,
symm_buffer,
backend.run_mega_moe(
y=y,
l1_weights=self._transformed_l1_weights,
l2_weights=self._transformed_l2_weights,
symm_buffer=symm_buffer,
activation_clamp=activation_clamp,
activation=self.activation,
fast_math=fast_math,
# DeepGEMM names the SiTU gate tanh scale `activation_alpha` and the
# linear/up tanh scale `activation_beta`; beta=0 leaves up untouched.
activation=self.activation,
activation_alpha=self.activation_beta or 1.0,
activation_beta=self.activation_linear_beta or 0.0,
fast_math=fast_math,
)
return y