[Performance] Optimize Dots3 NOTE runtime (#53517)

Signed-off-by: KurodaKanbei <mistergalahad@gmail.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
范裕达
2026-08-31 00:05:18 -07:00
committed by GitHub
co-authored by Jee Jee Li Jee Jee Li
parent 7292ee2791
commit da0b2d8b17
6 changed files with 94 additions and 85 deletions
+25 -22
View File
@@ -19,6 +19,7 @@ from vllm.distributed import (
tensor_model_parallel_all_reduce,
)
from vllm.model_executor.layers.attention import MLAAttention
from vllm.model_executor.layers.fused_embed_norm import has_full_vocab_on_rank
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -73,6 +74,28 @@ def _padded_mlp_size(
return ((blocks + tp_size - 1) // tp_size) * block_size[0] * tp_size
def _pad_dense_mlp_weight(
name: str,
loaded_weight: torch.Tensor,
weight_block_size: list[int] | None,
) -> torch.Tensor:
if weight_block_size is None or ".mlp.experts." in name:
return loaded_weight
if not any(
proj_name in name for proj_name in (".gate_proj.", ".up_proj.", ".down_proj.")
):
return loaded_weight
dim = 1 if ".down_proj." in name else 0
block_step = 1 if name.endswith("weight_scale_inv") else weight_block_size[0]
multiple = get_tensor_model_parallel_world_size() * block_step
pad = (-loaded_weight.shape[dim]) % multiple
if pad == 0:
return loaded_weight
pad_shape = list(loaded_weight.shape)
pad_shape[dim] = pad
return torch.cat([loaded_weight, loaded_weight.new_zeros(pad_shape)], dim=dim)
class Dots3NoteMoE(DeepseekV2MoE):
"""DeepSeek MoE with NOTE's block-padded shared expert."""
@@ -574,6 +597,7 @@ class Dots3NoteModel(DeepseekV32Model):
)
else:
self.embed_tokens = PPMissingLayer()
self.replicated_embed = has_full_vocab_on_rank(self.embed_tokens)
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers,
lambda prefix: Dots3NoteDecoderLayer(
@@ -595,27 +619,6 @@ class Dots3NoteModel(DeepseekV32Model):
vllm_config.parallel_config.eplb_config.num_redundant_experts
)
def _pad_dense_mlp_weight(
self, name: str, loaded_weight: torch.Tensor
) -> torch.Tensor:
block_size = self._weight_block_size
if block_size is None or ".mlp.experts." in name:
return loaded_weight
if not any(
proj_name in name
for proj_name in (".gate_proj.", ".up_proj.", ".down_proj.")
):
return loaded_weight
dim = 1 if ".down_proj." in name else 0
block_step = 1 if name.endswith("weight_scale_inv") else block_size[0]
multiple = get_tensor_model_parallel_world_size() * block_step
pad = (-loaded_weight.shape[dim]) % multiple
if pad == 0:
return loaded_weight
pad_shape = list(loaded_weight.shape)
pad_shape[dim] = pad
return torch.cat([loaded_weight, loaded_weight.new_zeros(pad_shape)], dim=dim)
def _adapt_weights(
self, weights: Iterable[tuple[str, torch.Tensor]]
) -> Iterator[tuple[str, torch.Tensor]]:
@@ -667,7 +670,7 @@ class Dots3NoteModel(DeepseekV32Model):
)
yield f"{layer_prefix}.{projection}.weight", dequantized
continue
yield name, self._pad_dense_mlp_weight(name, weight)
yield name, _pad_dense_mlp_weight(name, weight, self._weight_block_size)
if pending_indexer_fp8:
missing = ", ".join(
f"{prefix}.{proj}" for prefix, proj in pending_indexer_fp8