Fix Humming non-gated MoE (#49096)
This commit is contained in:
@@ -289,7 +289,14 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
assert a1.size(0) == num_experts
|
||||
num_tokens = a1.size(1)
|
||||
|
||||
return meta1.num_experts, num_tokens, meta1.shape_n // 2, meta1.shape_k, top_k
|
||||
return (
|
||||
meta1.num_experts,
|
||||
num_tokens,
|
||||
# Logical intermediate width for both gated and non-gated activations
|
||||
self.layer.intermediate_size_per_partition,
|
||||
meta1.shape_k,
|
||||
top_k,
|
||||
)
|
||||
|
||||
def get_buffer_metas(self, M: int, topk: int, activation: MoEActivation):
|
||||
from vllm.utils.humming import GemmType as HummingGemmType
|
||||
@@ -327,7 +334,8 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
real_shape_m = M * topk
|
||||
output_shape = (M, K)
|
||||
|
||||
down_input_size = N if activation.is_gated else (N * 2)
|
||||
gate_up_size = N * (2 if activation.is_gated else 1)
|
||||
down_input_size = N
|
||||
a_dtype = self.layer.humming_metas["w13"].a_dtype
|
||||
c_dtype = self.layer.humming_metas["w13"].c_dtype
|
||||
num_bits = a_dtype.num_bits
|
||||
@@ -347,7 +355,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
"dtype": torch_dtype_map[a_dtype],
|
||||
},
|
||||
"gate_up_output": {
|
||||
"shape": (real_shape_m, N * 2),
|
||||
"shape": (real_shape_m, gate_up_size),
|
||||
"dtype": torch_dtype_map[c_dtype],
|
||||
},
|
||||
"activation_output": {
|
||||
|
||||
@@ -788,7 +788,7 @@ def _convert_sublayer_to_humming(
|
||||
|
||||
shape_k_stacks = [shape_k]
|
||||
shape_n_stacks = [shape_n]
|
||||
if sublayer_name == "w13":
|
||||
if sublayer_name == "w13" and layer.moe_config.activation.is_gated:
|
||||
shape_n_stacks = [shape_n // 2] * 2
|
||||
|
||||
converted_weight_schema, converted_tensors = weight_schema.convert_humming(
|
||||
@@ -991,15 +991,15 @@ def convert_to_humming_moe_kernel_format(
|
||||
# Build sublayer configs from layer properties if not provided
|
||||
if sublayer_configs is None:
|
||||
is_gated = layer.moe_config.activation.is_gated
|
||||
intermediate_size = layer.moe_config.intermediate_size_per_partition
|
||||
sublayer_configs = {
|
||||
"w13": {
|
||||
"shape_n": layer.moe_config.intermediate_size_per_partition * 2,
|
||||
"shape_n": intermediate_size * (2 if is_gated else 1),
|
||||
"shape_k": layer.moe_config.hidden_dim,
|
||||
},
|
||||
"w2": {
|
||||
"shape_n": layer.moe_config.hidden_dim,
|
||||
"shape_k": layer.moe_config.intermediate_size_per_partition
|
||||
* (1 if is_gated else 2),
|
||||
"shape_k": intermediate_size,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user