[Performance] Optimize Dots3 NOTE runtime (#53517)
Signed-off-by: KurodaKanbei <mistergalahad@gmail.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
co-authored by
Jee Jee Li
Jee Jee Li
parent
7292ee2791
commit
da0b2d8b17
@@ -19,6 +19,7 @@ from vllm.distributed import (
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from vllm.model_executor.layers.attention import MLAAttention
|
||||
from vllm.model_executor.layers.fused_embed_norm import has_full_vocab_on_rank
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -73,6 +74,28 @@ def _padded_mlp_size(
|
||||
return ((blocks + tp_size - 1) // tp_size) * block_size[0] * tp_size
|
||||
|
||||
|
||||
def _pad_dense_mlp_weight(
|
||||
name: str,
|
||||
loaded_weight: torch.Tensor,
|
||||
weight_block_size: list[int] | None,
|
||||
) -> torch.Tensor:
|
||||
if weight_block_size is None or ".mlp.experts." in name:
|
||||
return loaded_weight
|
||||
if not any(
|
||||
proj_name in name for proj_name in (".gate_proj.", ".up_proj.", ".down_proj.")
|
||||
):
|
||||
return loaded_weight
|
||||
dim = 1 if ".down_proj." in name else 0
|
||||
block_step = 1 if name.endswith("weight_scale_inv") else weight_block_size[0]
|
||||
multiple = get_tensor_model_parallel_world_size() * block_step
|
||||
pad = (-loaded_weight.shape[dim]) % multiple
|
||||
if pad == 0:
|
||||
return loaded_weight
|
||||
pad_shape = list(loaded_weight.shape)
|
||||
pad_shape[dim] = pad
|
||||
return torch.cat([loaded_weight, loaded_weight.new_zeros(pad_shape)], dim=dim)
|
||||
|
||||
|
||||
class Dots3NoteMoE(DeepseekV2MoE):
|
||||
"""DeepSeek MoE with NOTE's block-padded shared expert."""
|
||||
|
||||
@@ -574,6 +597,7 @@ class Dots3NoteModel(DeepseekV32Model):
|
||||
)
|
||||
else:
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
self.replicated_embed = has_full_vocab_on_rank(self.embed_tokens)
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda prefix: Dots3NoteDecoderLayer(
|
||||
@@ -595,27 +619,6 @@ class Dots3NoteModel(DeepseekV32Model):
|
||||
vllm_config.parallel_config.eplb_config.num_redundant_experts
|
||||
)
|
||||
|
||||
def _pad_dense_mlp_weight(
|
||||
self, name: str, loaded_weight: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
block_size = self._weight_block_size
|
||||
if block_size is None or ".mlp.experts." in name:
|
||||
return loaded_weight
|
||||
if not any(
|
||||
proj_name in name
|
||||
for proj_name in (".gate_proj.", ".up_proj.", ".down_proj.")
|
||||
):
|
||||
return loaded_weight
|
||||
dim = 1 if ".down_proj." in name else 0
|
||||
block_step = 1 if name.endswith("weight_scale_inv") else block_size[0]
|
||||
multiple = get_tensor_model_parallel_world_size() * block_step
|
||||
pad = (-loaded_weight.shape[dim]) % multiple
|
||||
if pad == 0:
|
||||
return loaded_weight
|
||||
pad_shape = list(loaded_weight.shape)
|
||||
pad_shape[dim] = pad
|
||||
return torch.cat([loaded_weight, loaded_weight.new_zeros(pad_shape)], dim=dim)
|
||||
|
||||
def _adapt_weights(
|
||||
self, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterator[tuple[str, torch.Tensor]]:
|
||||
@@ -667,7 +670,7 @@ class Dots3NoteModel(DeepseekV32Model):
|
||||
)
|
||||
yield f"{layer_prefix}.{projection}.weight", dequantized
|
||||
continue
|
||||
yield name, self._pad_dense_mlp_weight(name, weight)
|
||||
yield name, _pad_dense_mlp_weight(name, weight, self._weight_block_size)
|
||||
if pending_indexer_fp8:
|
||||
missing = ", ".join(
|
||||
f"{prefix}.{proj}" for prefix, proj in pending_indexer_fp8
|
||||
|
||||
Reference in New Issue
Block a user