diff --git a/vllm/models/deepseek_v32/amd/model.py b/vllm/models/deepseek_v32/amd/model.py index f5067bd0b9..d2325372d9 100644 --- a/vllm/models/deepseek_v32/amd/model.py +++ b/vllm/models/deepseek_v32/amd/model.py @@ -12,6 +12,9 @@ from vllm.distributed import get_pp_group from vllm.model_executor.layers.fused_moe import ( fused_moe_make_expert_params_mapping, ) +from vllm.model_executor.layers.fused_moe.utils import ( + is_model_fused_shared_expert_compatible, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.vocab_parallel_embedding import ( VocabParallelEmbedding, @@ -158,6 +161,12 @@ class DeepseekV32Model(torch.nn.Module): prefix=f"{prefix}.layers", ) + self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible( + self.layers, + DeepseekV2MoE, + "mlp", + ) + if get_pp_group().is_last_rank: self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) else: @@ -226,7 +235,12 @@ class DeepseekV32Model(torch.nn.Module): ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", ckpt_up_proj_name="up_proj", - num_experts=self.config.n_routed_experts, + num_experts=self.config.n_routed_experts + + ( + self.config.n_shared_experts + if self.is_fused_shared_expert_enabled + else 0 + ), num_redundant_experts=self.num_redundant_experts, ) @@ -239,6 +253,9 @@ class DeepseekV32Model(torch.nn.Module): continue if get_spec_layer_idx_from_weight_name(self.config, name) is not None: continue + is_fusion_moe_shared_experts_layer = ( + self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name) + ) if _try_load_fp8_indexer_wk( name, loaded_weight, @@ -254,6 +271,8 @@ class DeepseekV32Model(torch.nn.Module): continue if ("mlp.experts." in name) and name not in params_dict: continue + if is_fusion_moe_shared_experts_layer: + continue name_mapped = name.replace(weight_name, param_name) if ( param_name == "fused_qkv_a_proj" @@ -269,43 +288,85 @@ class DeepseekV32Model(torch.nn.Module): break else: is_expert_weight = False - for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping # type: ignore[assignment] - if weight_name not in name: - continue - is_expert_weight = True - name_mapped = name.replace(weight_name, param_name) - if is_pp_missing_parameter(name_mapped, self): - continue - param = params_dict[name_mapped] - weight_loader = typing.cast( - Callable[..., bool], param.weight_loader + + # When AITER fusion_shared_experts is enabled the checkpoint + # provides a single widened shared_experts tensor per layer with + # no expert index. Split it evenly and route the slices into the + # expert slots appended after the routed experts. + num_chunks = 1 + if is_fusion_moe_shared_experts_layer: + num_chunks = getattr(self.config, "n_shared_experts", 1) or 1 + # gate/up are ColumnParallel (dim 0), down is RowParallel (dim 1). + split_dim = ( + 1 + if ("down_proj.weight" in name and loaded_weight.ndim > 1) + else 0 ) - success = weight_loader( - param, - loaded_weight, - name_mapped, - shard_id=shard_id, - expert_id=expert_id, - return_success=True, + total = loaded_weight.shape[split_dim] + assert total % num_chunks == 0, ( + f"Shared expert weight dim {total} " + f"not divisible by num_chunks {num_chunks}" ) - if success: - name = name_mapped - break - else: - if is_expert_weight: - continue - if name.endswith(".bias") and name not in params_dict: - continue - name = maybe_remap_kv_scale_name(name, params_dict) # type: ignore[assignment] - if name is None: - continue - if is_pp_missing_parameter(name, self): - continue - param = params_dict[name] - loader = getattr(param, "weight_loader", default_weight_loader) - loader(param, loaded_weight) - loaded_params.add(name) + chunk_size = total // num_chunks + + for j in range(num_chunks): + chunk_name = name + weight_to_load = loaded_weight + + if is_fusion_moe_shared_experts_layer: + chunk_slice = slice(j * chunk_size, (j + 1) * chunk_size) + if loaded_weight.ndim == 1: + weight_to_load = loaded_weight[chunk_slice] + elif split_dim == 0: + weight_to_load = loaded_weight[chunk_slice, :] + else: + weight_to_load = loaded_weight[:, chunk_slice] + chunk_name = name.replace( + "mlp.shared_experts", + f"mlp.experts.{self.config.n_routed_experts + j}", + ) + + for mapping in expert_params_mapping: + param_name, weight_name, expert_id, shard_id = mapping # type: ignore[assignment] + if weight_name not in chunk_name: + continue + is_expert_weight = True + name_mapped = chunk_name.replace(weight_name, param_name) + if is_pp_missing_parameter(name_mapped, self): + continue + param = params_dict[name_mapped] + weight_loader = typing.cast( + Callable[..., bool], param.weight_loader + ) + success = weight_loader( + param, + weight_to_load, + name_mapped, + shard_id=shard_id, + expert_id=expert_id, + return_success=True, + ) + if success: + if not is_fusion_moe_shared_experts_layer: + name = name_mapped + else: + loaded_params.add(name_mapped) + break + else: + if is_expert_weight: + continue + if name.endswith(".bias") and name not in params_dict: + continue + name = maybe_remap_kv_scale_name(name, params_dict) # type: ignore[assignment] + if name is None: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + loader = getattr(param, "weight_loader", default_weight_loader) + loader(param, loaded_weight) + if name is not None and not is_fusion_moe_shared_experts_layer: + loaded_params.add(name) return loaded_params diff --git a/vllm/models/deepseek_v32/amd/mtp.py b/vllm/models/deepseek_v32/amd/mtp.py index 46c35dec98..70ddeae374 100644 --- a/vllm/models/deepseek_v32/amd/mtp.py +++ b/vllm/models/deepseek_v32/amd/mtp.py @@ -130,6 +130,15 @@ class DeepseekV32MultiTokenPredictor(nn.Module): if self_attn is not None and hasattr(self_attn, "skip_topk"): self_attn.skip_topk = skip + def compact_topk_indices(self, slot_ids: torch.Tensor): + """Gather the top-k index rows at ``slot_ids`` to the front of the buffer.""" + num_slots = slot_ids.numel() + for layer in self.layers.values(): + self_attn = getattr(layer.mtp_block, "self_attn", None) + if self_attn is not None and hasattr(self_attn, "topk_indices_buffer"): + topk_indices_buffer = self_attn.topk_indices_buffer + topk_indices_buffer[:num_slots] = topk_indices_buffer[slot_ids] + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.embed_tokens(input_ids)