[DeepSeek-V4] Support TEP=16 for the block-FP8 shared expert (#46001)

Signed-off-by: Jeff Ma <jeffjma@umich.edu>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Jeff (Junze) Ma
2026-06-18 20:10:12 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent ab66606993
commit 2a6c6b9429
+44
View File
@@ -64,6 +64,7 @@ from vllm.models.deepseek_v4.nvidia.flashinfer_sparse import (
from vllm.models.deepseek_v4.nvidia.flashmla import DeepseekV4FlashMLAAttention
from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_inputs
from vllm.sequence import IntermediateTensors
from vllm.utils.math_utils import cdiv
from vllm.v1.attention.backends.registry import AttentionBackendEnum
@@ -85,6 +86,15 @@ class DeepseekV4MLP(nn.Module):
# across the ranks within the tp_group. In this case the weights are
# replicated and no collective ops are needed.
# Otherwise we use standard TP with an allreduce at the end.
#
# Block-FP8 shards in whole 128-blocks; cdiv rounds the per-rank block
# count up so the linear's even TP split stays block-aligned, with the
# trailing ranks zero-filled by load_weights.
block_size = getattr(quant_config, "weight_block_size", None)
if block_size is not None and not is_sequence_parallel:
tp_size = get_tensor_model_parallel_world_size()
n_local = cdiv(intermediate_size // block_size[0], tp_size)
intermediate_size = n_local * block_size[0] * tp_size
self.gate_up_proj = MergedColumnParallelLinear(
hidden_size,
[intermediate_size] * 2,
@@ -892,6 +902,8 @@ class DeepseekV4Model(nn.Module):
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
self.config = config
self.quant_config = quant_config
self.parallel_config = vllm_config.parallel_config
self.use_mega_moe = (
vllm_config.kernel_config.moe_backend == "deep_gemm_mega_moe"
)
@@ -1080,7 +1092,17 @@ class DeepseekV4Model(nn.Module):
# Pre-compute expert mapping ONCE.
expert_mapping = self.get_expert_mapping()
# Block-FP8 shared experts: pad the intermediate up to the TP-uniform
# block count so the standard loaders below slice it evenly (trailing
# ranks land on the zero pad). SP / unquantized ones need no padding.
pad_shared_expert = (
getattr(self.quant_config, "weight_block_size", None) is not None
and not self.parallel_config.use_sequence_parallel_moe
)
for name, loaded_weight in weights:
if pad_shared_expert and ".shared_experts." in name:
loaded_weight = self._pad_shared_expert_weight(name, loaded_weight)
for param_name, weight_name, shard_id in stacked_params_mapping:
# Skip non-stacked layers and experts (experts handled below).
if ".experts." in name:
@@ -1155,6 +1177,28 @@ class DeepseekV4Model(nn.Module):
return loaded_params
def _pad_shared_expert_weight(
self, name: str, loaded_weight: torch.Tensor
) -> torch.Tensor:
"""Zero-pad a block-FP8 shared-expert weight/scale on its intermediate
axis so the standard TP loaders split it into even, block-aligned shards
(trailing ranks get the zero pad). gate (w1)/up (w3) [I, H] pad dim 0;
down (w2 -> down_proj) [H, I] pads dim 1.
"""
block_size = getattr(self.quant_config, "weight_block_size", None)
assert block_size is not None
# Round the intermediate axis up to a whole number of TP shards. The axis
# is in elements for weights (step = block) and in blocks for scales.
step = 1 if name.endswith("weight_scale_inv") else block_size[0]
dim = 1 if ".down_proj." in name else 0
mult = get_tensor_model_parallel_world_size() * step
pad = cdiv(loaded_weight.shape[dim], mult) * mult - loaded_weight.shape[dim]
if pad == 0:
return loaded_weight
pad_shape = list(loaded_weight.shape)
pad_shape[dim] = pad
return torch.cat([loaded_weight, loaded_weight.new_zeros(pad_shape)], dim=dim)
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
first_layer = next(iter(islice(self.layers, self.start_layer, self.end_layer)))
if first_layer.ffn.use_mega_moe: