[DeepSeek-V4] Support TEP=16 for the block-FP8 shared expert (#46001)
Signed-off-by: Jeff Ma <jeffjma@umich.edu> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
ab66606993
commit
2a6c6b9429
@@ -64,6 +64,7 @@ from vllm.models.deepseek_v4.nvidia.flashinfer_sparse import (
|
||||
from vllm.models.deepseek_v4.nvidia.flashmla import DeepseekV4FlashMLAAttention
|
||||
from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_inputs
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
|
||||
@@ -85,6 +86,15 @@ class DeepseekV4MLP(nn.Module):
|
||||
# across the ranks within the tp_group. In this case the weights are
|
||||
# replicated and no collective ops are needed.
|
||||
# Otherwise we use standard TP with an allreduce at the end.
|
||||
#
|
||||
# Block-FP8 shards in whole 128-blocks; cdiv rounds the per-rank block
|
||||
# count up so the linear's even TP split stays block-aligned, with the
|
||||
# trailing ranks zero-filled by load_weights.
|
||||
block_size = getattr(quant_config, "weight_block_size", None)
|
||||
if block_size is not None and not is_sequence_parallel:
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
n_local = cdiv(intermediate_size // block_size[0], tp_size)
|
||||
intermediate_size = n_local * block_size[0] * tp_size
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
hidden_size,
|
||||
[intermediate_size] * 2,
|
||||
@@ -892,6 +902,8 @@ class DeepseekV4Model(nn.Module):
|
||||
config = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
self.parallel_config = vllm_config.parallel_config
|
||||
self.use_mega_moe = (
|
||||
vllm_config.kernel_config.moe_backend == "deep_gemm_mega_moe"
|
||||
)
|
||||
@@ -1080,7 +1092,17 @@ class DeepseekV4Model(nn.Module):
|
||||
# Pre-compute expert mapping ONCE.
|
||||
expert_mapping = self.get_expert_mapping()
|
||||
|
||||
# Block-FP8 shared experts: pad the intermediate up to the TP-uniform
|
||||
# block count so the standard loaders below slice it evenly (trailing
|
||||
# ranks land on the zero pad). SP / unquantized ones need no padding.
|
||||
pad_shared_expert = (
|
||||
getattr(self.quant_config, "weight_block_size", None) is not None
|
||||
and not self.parallel_config.use_sequence_parallel_moe
|
||||
)
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
if pad_shared_expert and ".shared_experts." in name:
|
||||
loaded_weight = self._pad_shared_expert_weight(name, loaded_weight)
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
# Skip non-stacked layers and experts (experts handled below).
|
||||
if ".experts." in name:
|
||||
@@ -1155,6 +1177,28 @@ class DeepseekV4Model(nn.Module):
|
||||
|
||||
return loaded_params
|
||||
|
||||
def _pad_shared_expert_weight(
|
||||
self, name: str, loaded_weight: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Zero-pad a block-FP8 shared-expert weight/scale on its intermediate
|
||||
axis so the standard TP loaders split it into even, block-aligned shards
|
||||
(trailing ranks get the zero pad). gate (w1)/up (w3) [I, H] pad dim 0;
|
||||
down (w2 -> down_proj) [H, I] pads dim 1.
|
||||
"""
|
||||
block_size = getattr(self.quant_config, "weight_block_size", None)
|
||||
assert block_size is not None
|
||||
# Round the intermediate axis up to a whole number of TP shards. The axis
|
||||
# is in elements for weights (step = block) and in blocks for scales.
|
||||
step = 1 if name.endswith("weight_scale_inv") else block_size[0]
|
||||
dim = 1 if ".down_proj." in name else 0
|
||||
mult = get_tensor_model_parallel_world_size() * step
|
||||
pad = cdiv(loaded_weight.shape[dim], mult) * mult - loaded_weight.shape[dim]
|
||||
if pad == 0:
|
||||
return loaded_weight
|
||||
pad_shape = list(loaded_weight.shape)
|
||||
pad_shape[dim] = pad
|
||||
return torch.cat([loaded_weight, loaded_weight.new_zeros(pad_shape)], dim=dim)
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
first_layer = next(iter(islice(self.layers, self.start_layer, self.end_layer)))
|
||||
if first_layer.ffn.use_mega_moe:
|
||||
|
||||
Reference in New Issue
Block a user