diff --git a/vllm/models/deepseek_v4/nvidia/model.py b/vllm/models/deepseek_v4/nvidia/model.py index 364754f9d7..868fc3f5fd 100644 --- a/vllm/models/deepseek_v4/nvidia/model.py +++ b/vllm/models/deepseek_v4/nvidia/model.py @@ -64,6 +64,7 @@ from vllm.models.deepseek_v4.nvidia.flashinfer_sparse import ( from vllm.models.deepseek_v4.nvidia.flashmla import DeepseekV4FlashMLAAttention from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_inputs from vllm.sequence import IntermediateTensors +from vllm.utils.math_utils import cdiv from vllm.v1.attention.backends.registry import AttentionBackendEnum @@ -85,6 +86,15 @@ class DeepseekV4MLP(nn.Module): # across the ranks within the tp_group. In this case the weights are # replicated and no collective ops are needed. # Otherwise we use standard TP with an allreduce at the end. + # + # Block-FP8 shards in whole 128-blocks; cdiv rounds the per-rank block + # count up so the linear's even TP split stays block-aligned, with the + # trailing ranks zero-filled by load_weights. + block_size = getattr(quant_config, "weight_block_size", None) + if block_size is not None and not is_sequence_parallel: + tp_size = get_tensor_model_parallel_world_size() + n_local = cdiv(intermediate_size // block_size[0], tp_size) + intermediate_size = n_local * block_size[0] * tp_size self.gate_up_proj = MergedColumnParallelLinear( hidden_size, [intermediate_size] * 2, @@ -892,6 +902,8 @@ class DeepseekV4Model(nn.Module): config = vllm_config.model_config.hf_config quant_config = vllm_config.quant_config self.config = config + self.quant_config = quant_config + self.parallel_config = vllm_config.parallel_config self.use_mega_moe = ( vllm_config.kernel_config.moe_backend == "deep_gemm_mega_moe" ) @@ -1080,7 +1092,17 @@ class DeepseekV4Model(nn.Module): # Pre-compute expert mapping ONCE. expert_mapping = self.get_expert_mapping() + # Block-FP8 shared experts: pad the intermediate up to the TP-uniform + # block count so the standard loaders below slice it evenly (trailing + # ranks land on the zero pad). SP / unquantized ones need no padding. + pad_shared_expert = ( + getattr(self.quant_config, "weight_block_size", None) is not None + and not self.parallel_config.use_sequence_parallel_moe + ) + for name, loaded_weight in weights: + if pad_shared_expert and ".shared_experts." in name: + loaded_weight = self._pad_shared_expert_weight(name, loaded_weight) for param_name, weight_name, shard_id in stacked_params_mapping: # Skip non-stacked layers and experts (experts handled below). if ".experts." in name: @@ -1155,6 +1177,28 @@ class DeepseekV4Model(nn.Module): return loaded_params + def _pad_shared_expert_weight( + self, name: str, loaded_weight: torch.Tensor + ) -> torch.Tensor: + """Zero-pad a block-FP8 shared-expert weight/scale on its intermediate + axis so the standard TP loaders split it into even, block-aligned shards + (trailing ranks get the zero pad). gate (w1)/up (w3) [I, H] pad dim 0; + down (w2 -> down_proj) [H, I] pads dim 1. + """ + block_size = getattr(self.quant_config, "weight_block_size", None) + assert block_size is not None + # Round the intermediate axis up to a whole number of TP shards. The axis + # is in elements for weights (step = block) and in blocks for scales. + step = 1 if name.endswith("weight_scale_inv") else block_size[0] + dim = 1 if ".down_proj." in name else 0 + mult = get_tensor_model_parallel_world_size() * step + pad = cdiv(loaded_weight.shape[dim], mult) * mult - loaded_weight.shape[dim] + if pad == 0: + return loaded_weight + pad_shape = list(loaded_weight.shape) + pad_shape[dim] = pad + return torch.cat([loaded_weight, loaded_weight.new_zeros(pad_shape)], dim=dim) + def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: first_layer = next(iter(islice(self.layers, self.start_layer, self.end_layer))) if first_layer.ffn.use_mega_moe: