[Refactor] Abstract DeepGemm MegaMOE backend (#58053)
Signed-off-by: Hui Shi <kalinshi@tencent.com> Co-authored-by: LuneExplorer <liyihan23@mails.tsinghua.edu.cn> Co-authored-by: ArmageddonKnight <bojian@cs.toronto.edu>
This commit is contained in:
co-authored by
LuneExplorer
ArmageddonKnight
parent
4ac0179f8a
commit
81198e97ba
@@ -365,27 +365,16 @@ class KimiK3MegaMoEExperts(DeepseekV4MegaMoEExperts):
|
||||
return
|
||||
|
||||
self._check_runtime_supported()
|
||||
from vllm.utils.deep_gemm import _import_deep_gemm
|
||||
|
||||
deep_gemm = _import_deep_gemm()
|
||||
w13_scale = deep_gemm.transform_sf_into_required_layout(
|
||||
self._ue8m0_uint8_to_float(self.w13_weight_scale.data).contiguous(),
|
||||
2 * self.intermediate_size,
|
||||
self.hidden_size,
|
||||
(1, 32),
|
||||
self.num_local_experts,
|
||||
)
|
||||
w2_scale = deep_gemm.transform_sf_into_required_layout(
|
||||
self._ue8m0_uint8_to_float(self.w2_weight_scale.data).contiguous(),
|
||||
self.hidden_size,
|
||||
self.intermediate_size,
|
||||
(1, 32),
|
||||
self.num_local_experts,
|
||||
)
|
||||
backend = self._ensure_backend()
|
||||
self._transformed_l1_weights, self._transformed_l2_weights = (
|
||||
deep_gemm.transform_weights_for_mega_moe(
|
||||
(self.w13_weight.data.view(torch.int8).contiguous(), w13_scale),
|
||||
(self.w2_weight.data.view(torch.int8).contiguous(), w2_scale),
|
||||
backend.transform_weights(
|
||||
w13_weight=self.w13_weight.data,
|
||||
w13_weight_scale=self.w13_weight_scale.data,
|
||||
w2_weight=self.w2_weight.data,
|
||||
w2_weight_scale=self.w2_weight_scale.data,
|
||||
num_local_experts=self.num_local_experts,
|
||||
hidden_size=self.hidden_size,
|
||||
intermediate_size=self.intermediate_size,
|
||||
activation=self.activation,
|
||||
)
|
||||
)
|
||||
@@ -407,6 +396,7 @@ class KimiK3MegaMoEExperts(DeepseekV4MegaMoEExperts):
|
||||
from vllm.utils.deep_gemm import _import_deep_gemm
|
||||
|
||||
deep_gemm = _import_deep_gemm()
|
||||
backend = self._ensure_backend()
|
||||
group = get_ep_group().device_group
|
||||
device = torch.accelerator.current_device_index()
|
||||
key = (
|
||||
@@ -418,6 +408,7 @@ class KimiK3MegaMoEExperts(DeepseekV4MegaMoEExperts):
|
||||
self.hidden_size,
|
||||
self.intermediate_size,
|
||||
self.activation,
|
||||
backend.mma_type,
|
||||
)
|
||||
symm_buffer = self._kimi_symm_buffer_cache.get(key)
|
||||
if symm_buffer is None:
|
||||
@@ -429,6 +420,7 @@ class KimiK3MegaMoEExperts(DeepseekV4MegaMoEExperts):
|
||||
self.hidden_size,
|
||||
self.intermediate_size,
|
||||
activation=self.activation,
|
||||
mma_type=backend.mma_type,
|
||||
)
|
||||
self._kimi_symm_buffer_cache[key] = symm_buffer
|
||||
return symm_buffer
|
||||
@@ -449,10 +441,6 @@ class KimiK3MegaMoEExperts(DeepseekV4MegaMoEExperts):
|
||||
f"but its symmetric buffer supports {self.max_num_tokens}."
|
||||
)
|
||||
y = torch.empty_like(hidden_states, dtype=torch.bfloat16)
|
||||
from vllm.utils.deep_gemm import _import_deep_gemm
|
||||
|
||||
deep_gemm = _import_deep_gemm()
|
||||
symm_buffer = self.get_symm_buffer()
|
||||
num_tokens = hidden_states.shape[0]
|
||||
is_padding = None
|
||||
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
|
||||
@@ -483,6 +471,8 @@ class KimiK3MegaMoEExperts(DeepseekV4MegaMoEExperts):
|
||||
else None,
|
||||
)
|
||||
|
||||
backend = self._ensure_backend()
|
||||
symm_buffer = self.get_symm_buffer()
|
||||
prepare_megamoe_inputs(
|
||||
hidden_states,
|
||||
topk_weights,
|
||||
@@ -492,22 +482,23 @@ class KimiK3MegaMoEExperts(DeepseekV4MegaMoEExperts):
|
||||
symm_buffer.topk_idx[:num_tokens],
|
||||
symm_buffer.topk_weights[:num_tokens],
|
||||
is_padding=is_padding,
|
||||
hidden_quant=backend.hidden_quant,
|
||||
)
|
||||
self.finalize_weights()
|
||||
assert self._transformed_l1_weights is not None
|
||||
assert self._transformed_l2_weights is not None
|
||||
deep_gemm.fp8_fp4_mega_moe(
|
||||
y,
|
||||
self._transformed_l1_weights,
|
||||
self._transformed_l2_weights,
|
||||
symm_buffer,
|
||||
backend.run_mega_moe(
|
||||
y=y,
|
||||
l1_weights=self._transformed_l1_weights,
|
||||
l2_weights=self._transformed_l2_weights,
|
||||
symm_buffer=symm_buffer,
|
||||
activation_clamp=activation_clamp,
|
||||
activation=self.activation,
|
||||
fast_math=fast_math,
|
||||
# DeepGEMM names the SiTU gate tanh scale `activation_alpha` and the
|
||||
# linear/up tanh scale `activation_beta`; beta=0 leaves up untouched.
|
||||
activation=self.activation,
|
||||
activation_alpha=self.activation_beta or 1.0,
|
||||
activation_beta=self.activation_linear_beta or 0.0,
|
||||
fast_math=fast_math,
|
||||
)
|
||||
return y
|
||||
|
||||
|
||||
Reference in New Issue
Block a user