[ROCm][Model][Bugfix] Fix GLM-5.2 shared-expert fusion and MTP on the ROCm DSA path (#58904)
Signed-off-by: Jin Tao <jin.tao@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
co-authored by
Cursor
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Shanshan Shen
parent
30f5c01d4e
commit
77d9cda688
@@ -12,6 +12,9 @@ from vllm.distributed import get_pp_group
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
@@ -158,6 +161,12 @@ class DeepseekV32Model(torch.nn.Module):
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
|
||||
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
|
||||
self.layers,
|
||||
DeepseekV2MoE,
|
||||
"mlp",
|
||||
)
|
||||
|
||||
if get_pp_group().is_last_rank:
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
else:
|
||||
@@ -226,7 +235,12 @@ class DeepseekV32Model(torch.nn.Module):
|
||||
ckpt_gate_proj_name="gate_proj",
|
||||
ckpt_down_proj_name="down_proj",
|
||||
ckpt_up_proj_name="up_proj",
|
||||
num_experts=self.config.n_routed_experts,
|
||||
num_experts=self.config.n_routed_experts
|
||||
+ (
|
||||
self.config.n_shared_experts
|
||||
if self.is_fused_shared_expert_enabled
|
||||
else 0
|
||||
),
|
||||
num_redundant_experts=self.num_redundant_experts,
|
||||
)
|
||||
|
||||
@@ -239,6 +253,9 @@ class DeepseekV32Model(torch.nn.Module):
|
||||
continue
|
||||
if get_spec_layer_idx_from_weight_name(self.config, name) is not None:
|
||||
continue
|
||||
is_fusion_moe_shared_experts_layer = (
|
||||
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
|
||||
)
|
||||
if _try_load_fp8_indexer_wk(
|
||||
name,
|
||||
loaded_weight,
|
||||
@@ -254,6 +271,8 @@ class DeepseekV32Model(torch.nn.Module):
|
||||
continue
|
||||
if ("mlp.experts." in name) and name not in params_dict:
|
||||
continue
|
||||
if is_fusion_moe_shared_experts_layer:
|
||||
continue
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
if (
|
||||
param_name == "fused_qkv_a_proj"
|
||||
@@ -269,43 +288,85 @@ class DeepseekV32Model(torch.nn.Module):
|
||||
break
|
||||
else:
|
||||
is_expert_weight = False
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping # type: ignore[assignment]
|
||||
if weight_name not in name:
|
||||
continue
|
||||
is_expert_weight = True
|
||||
name_mapped = name.replace(weight_name, param_name)
|
||||
if is_pp_missing_parameter(name_mapped, self):
|
||||
continue
|
||||
param = params_dict[name_mapped]
|
||||
weight_loader = typing.cast(
|
||||
Callable[..., bool], param.weight_loader
|
||||
|
||||
# When AITER fusion_shared_experts is enabled the checkpoint
|
||||
# provides a single widened shared_experts tensor per layer with
|
||||
# no expert index. Split it evenly and route the slices into the
|
||||
# expert slots appended after the routed experts.
|
||||
num_chunks = 1
|
||||
if is_fusion_moe_shared_experts_layer:
|
||||
num_chunks = getattr(self.config, "n_shared_experts", 1) or 1
|
||||
# gate/up are ColumnParallel (dim 0), down is RowParallel (dim 1).
|
||||
split_dim = (
|
||||
1
|
||||
if ("down_proj.weight" in name and loaded_weight.ndim > 1)
|
||||
else 0
|
||||
)
|
||||
success = weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
name_mapped,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
total = loaded_weight.shape[split_dim]
|
||||
assert total % num_chunks == 0, (
|
||||
f"Shared expert weight dim {total} "
|
||||
f"not divisible by num_chunks {num_chunks}"
|
||||
)
|
||||
if success:
|
||||
name = name_mapped
|
||||
break
|
||||
else:
|
||||
if is_expert_weight:
|
||||
continue
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
name = maybe_remap_kv_scale_name(name, params_dict) # type: ignore[assignment]
|
||||
if name is None:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
chunk_size = total // num_chunks
|
||||
|
||||
for j in range(num_chunks):
|
||||
chunk_name = name
|
||||
weight_to_load = loaded_weight
|
||||
|
||||
if is_fusion_moe_shared_experts_layer:
|
||||
chunk_slice = slice(j * chunk_size, (j + 1) * chunk_size)
|
||||
if loaded_weight.ndim == 1:
|
||||
weight_to_load = loaded_weight[chunk_slice]
|
||||
elif split_dim == 0:
|
||||
weight_to_load = loaded_weight[chunk_slice, :]
|
||||
else:
|
||||
weight_to_load = loaded_weight[:, chunk_slice]
|
||||
chunk_name = name.replace(
|
||||
"mlp.shared_experts",
|
||||
f"mlp.experts.{self.config.n_routed_experts + j}",
|
||||
)
|
||||
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping # type: ignore[assignment]
|
||||
if weight_name not in chunk_name:
|
||||
continue
|
||||
is_expert_weight = True
|
||||
name_mapped = chunk_name.replace(weight_name, param_name)
|
||||
if is_pp_missing_parameter(name_mapped, self):
|
||||
continue
|
||||
param = params_dict[name_mapped]
|
||||
weight_loader = typing.cast(
|
||||
Callable[..., bool], param.weight_loader
|
||||
)
|
||||
success = weight_loader(
|
||||
param,
|
||||
weight_to_load,
|
||||
name_mapped,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
if not is_fusion_moe_shared_experts_layer:
|
||||
name = name_mapped
|
||||
else:
|
||||
loaded_params.add(name_mapped)
|
||||
break
|
||||
else:
|
||||
if is_expert_weight:
|
||||
continue
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
name = maybe_remap_kv_scale_name(name, params_dict) # type: ignore[assignment]
|
||||
if name is None:
|
||||
continue
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
loader(param, loaded_weight)
|
||||
if name is not None and not is_fusion_moe_shared_experts_layer:
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
|
||||
|
||||
|
||||
@@ -130,6 +130,15 @@ class DeepseekV32MultiTokenPredictor(nn.Module):
|
||||
if self_attn is not None and hasattr(self_attn, "skip_topk"):
|
||||
self_attn.skip_topk = skip
|
||||
|
||||
def compact_topk_indices(self, slot_ids: torch.Tensor):
|
||||
"""Gather the top-k index rows at ``slot_ids`` to the front of the buffer."""
|
||||
num_slots = slot_ids.numel()
|
||||
for layer in self.layers.values():
|
||||
self_attn = getattr(layer.mtp_block, "self_attn", None)
|
||||
if self_attn is not None and hasattr(self_attn, "topk_indices_buffer"):
|
||||
topk_indices_buffer = self_attn.topk_indices_buffer
|
||||
topk_indices_buffer[:num_slots] = topk_indices_buffer[slot_ids]
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user