[ROCm][Model][Bugfix] Fix GLM-5.2 shared-expert fusion and MTP on the ROCm DSA path (#58904)

Signed-off-by: Jin Tao <jin.tao@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
Jin Tao
2026-09-29 07:57:23 +00:00
committed by GitHub
co-authored by Cursor mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Shanshan Shen
parent 30f5c01d4e
commit 77d9cda688
2 changed files with 106 additions and 36 deletions
+97 -36
View File
@@ -12,6 +12,9 @@ from vllm.distributed import get_pp_group
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
@@ -158,6 +161,12 @@ class DeepseekV32Model(torch.nn.Module):
prefix=f"{prefix}.layers",
)
self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible(
self.layers,
DeepseekV2MoE,
"mlp",
)
if get_pp_group().is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
else:
@@ -226,7 +235,12 @@ class DeepseekV32Model(torch.nn.Module):
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.n_routed_experts,
num_experts=self.config.n_routed_experts
+ (
self.config.n_shared_experts
if self.is_fused_shared_expert_enabled
else 0
),
num_redundant_experts=self.num_redundant_experts,
)
@@ -239,6 +253,9 @@ class DeepseekV32Model(torch.nn.Module):
continue
if get_spec_layer_idx_from_weight_name(self.config, name) is not None:
continue
is_fusion_moe_shared_experts_layer = (
self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name)
)
if _try_load_fp8_indexer_wk(
name,
loaded_weight,
@@ -254,6 +271,8 @@ class DeepseekV32Model(torch.nn.Module):
continue
if ("mlp.experts." in name) and name not in params_dict:
continue
if is_fusion_moe_shared_experts_layer:
continue
name_mapped = name.replace(weight_name, param_name)
if (
param_name == "fused_qkv_a_proj"
@@ -269,43 +288,85 @@ class DeepseekV32Model(torch.nn.Module):
break
else:
is_expert_weight = False
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping # type: ignore[assignment]
if weight_name not in name:
continue
is_expert_weight = True
name_mapped = name.replace(weight_name, param_name)
if is_pp_missing_parameter(name_mapped, self):
continue
param = params_dict[name_mapped]
weight_loader = typing.cast(
Callable[..., bool], param.weight_loader
# When AITER fusion_shared_experts is enabled the checkpoint
# provides a single widened shared_experts tensor per layer with
# no expert index. Split it evenly and route the slices into the
# expert slots appended after the routed experts.
num_chunks = 1
if is_fusion_moe_shared_experts_layer:
num_chunks = getattr(self.config, "n_shared_experts", 1) or 1
# gate/up are ColumnParallel (dim 0), down is RowParallel (dim 1).
split_dim = (
1
if ("down_proj.weight" in name and loaded_weight.ndim > 1)
else 0
)
success = weight_loader(
param,
loaded_weight,
name_mapped,
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
total = loaded_weight.shape[split_dim]
assert total % num_chunks == 0, (
f"Shared expert weight dim {total} "
f"not divisible by num_chunks {num_chunks}"
)
if success:
name = name_mapped
break
else:
if is_expert_weight:
continue
if name.endswith(".bias") and name not in params_dict:
continue
name = maybe_remap_kv_scale_name(name, params_dict) # type: ignore[assignment]
if name is None:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
loader = getattr(param, "weight_loader", default_weight_loader)
loader(param, loaded_weight)
loaded_params.add(name)
chunk_size = total // num_chunks
for j in range(num_chunks):
chunk_name = name
weight_to_load = loaded_weight
if is_fusion_moe_shared_experts_layer:
chunk_slice = slice(j * chunk_size, (j + 1) * chunk_size)
if loaded_weight.ndim == 1:
weight_to_load = loaded_weight[chunk_slice]
elif split_dim == 0:
weight_to_load = loaded_weight[chunk_slice, :]
else:
weight_to_load = loaded_weight[:, chunk_slice]
chunk_name = name.replace(
"mlp.shared_experts",
f"mlp.experts.{self.config.n_routed_experts + j}",
)
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping # type: ignore[assignment]
if weight_name not in chunk_name:
continue
is_expert_weight = True
name_mapped = chunk_name.replace(weight_name, param_name)
if is_pp_missing_parameter(name_mapped, self):
continue
param = params_dict[name_mapped]
weight_loader = typing.cast(
Callable[..., bool], param.weight_loader
)
success = weight_loader(
param,
weight_to_load,
name_mapped,
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
)
if success:
if not is_fusion_moe_shared_experts_layer:
name = name_mapped
else:
loaded_params.add(name_mapped)
break
else:
if is_expert_weight:
continue
if name.endswith(".bias") and name not in params_dict:
continue
name = maybe_remap_kv_scale_name(name, params_dict) # type: ignore[assignment]
if name is None:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
loader = getattr(param, "weight_loader", default_weight_loader)
loader(param, loaded_weight)
if name is not None and not is_fusion_moe_shared_experts_layer:
loaded_params.add(name)
return loaded_params
+9
View File
@@ -130,6 +130,15 @@ class DeepseekV32MultiTokenPredictor(nn.Module):
if self_attn is not None and hasattr(self_attn, "skip_topk"):
self_attn.skip_topk = skip
def compact_topk_indices(self, slot_ids: torch.Tensor):
"""Gather the top-k index rows at ``slot_ids`` to the front of the buffer."""
num_slots = slot_ids.numel()
for layer in self.layers.values():
self_attn = getattr(layer.mtp_block, "self_attn", None)
if self_attn is not None and hasattr(self_attn, "topk_indices_buffer"):
topk_indices_buffer = self_attn.topk_indices_buffer
topk_indices_buffer[:num_slots] = topk_indices_buffer[slot_ids]
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)