Replicated embedding and norm fusion for DSV3 flat model (#48484)
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
@@ -10,13 +10,15 @@ import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_pp_group
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.model_executor.layers.fused_embed_norm import (
|
||||
fused_embed_norm,
|
||||
has_full_vocab_on_rank,
|
||||
make_input_embedding,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
default_weight_loader,
|
||||
maybe_remap_kv_scale_name,
|
||||
@@ -116,12 +118,18 @@ class DeepseekV32DecoderLayer(torch.nn.Module):
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
attn_in: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
full_num_tokens = positions.shape[0]
|
||||
|
||||
if residual is None:
|
||||
# First layer: hidden_states is the embedding (the residual).
|
||||
residual = hidden_states
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
# ``attn_in`` is input_layernorm(embedding) already computed fused
|
||||
# with the embedding gather; otherwise apply it here.
|
||||
hidden_states = (
|
||||
attn_in if attn_in is not None else self.input_layernorm(hidden_states)
|
||||
)
|
||||
elif self.use_sequence_parallel:
|
||||
hidden_states, residual = self.input_layernorm(hidden_states, residual)
|
||||
else:
|
||||
@@ -182,14 +190,17 @@ class DeepseekV32Model(torch.nn.Module):
|
||||
)
|
||||
|
||||
if get_pp_group().is_first_rank:
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
self.embed_tokens = make_input_embedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.embed_tokens",
|
||||
tie_word_embeddings=getattr(config, "tie_word_embeddings", False),
|
||||
)
|
||||
else:
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
# The fused embed+norm gather needs the full table on-rank.
|
||||
self.replicated_embed = has_full_vocab_on_rank(self.embed_tokens)
|
||||
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.num_hidden_layers,
|
||||
@@ -222,9 +233,21 @@ class DeepseekV32Model(torch.nn.Module):
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
attn_in = None
|
||||
if get_pp_group().is_first_rank:
|
||||
if inputs_embeds is not None:
|
||||
hidden_states = inputs_embeds
|
||||
elif self.replicated_embed:
|
||||
assert input_ids is not None
|
||||
# Full table on-rank: gather the embedding and the first layer's
|
||||
# input_layernorm in one launch. ``attn_in`` is the pre-normed
|
||||
# attention input; ``hidden_states`` is the (residual) embedding.
|
||||
hidden_states, attn_in = fused_embed_norm(
|
||||
input_ids,
|
||||
self.embed_tokens.weight,
|
||||
chain_weight=self.layers[self.start_layer].input_layernorm.weight,
|
||||
eps=self.config.rms_norm_eps,
|
||||
)
|
||||
else:
|
||||
assert input_ids is not None
|
||||
hidden_states = self.embed_input_ids(input_ids)
|
||||
@@ -242,6 +265,8 @@ class DeepseekV32Model(torch.nn.Module):
|
||||
forward_context.is_padding, hidden_states
|
||||
)
|
||||
hidden_states = sp_shard(hidden_states)
|
||||
if attn_in is not None:
|
||||
attn_in = sp_shard(attn_in)
|
||||
assert residual is None, "Currently, SP is not supported with PP"
|
||||
|
||||
aux_hidden_states = []
|
||||
@@ -253,7 +278,8 @@ class DeepseekV32Model(torch.nn.Module):
|
||||
aux_hidden_states.append(
|
||||
hidden_states if residual is None else hidden_states + residual
|
||||
)
|
||||
hidden_states, residual = layer(positions, hidden_states, residual)
|
||||
hidden_states, residual = layer(positions, hidden_states, residual, attn_in)
|
||||
attn_in = None
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
assert not self.use_sequence_parallel, (
|
||||
|
||||
@@ -10,14 +10,16 @@ import vllm.envs as envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.model_executor.layers.fused_embed_norm import (
|
||||
fused_embed_eh_norm,
|
||||
has_full_vocab_on_rank,
|
||||
make_input_embedding,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.mtp_validation import (
|
||||
is_mtp_completeness_check_enabled,
|
||||
)
|
||||
@@ -93,18 +95,33 @@ class DeepseekV32MultiTokenPredictorLayer(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
embed_table: torch.Tensor | None = None,
|
||||
spec_step_index: int = 0,
|
||||
) -> torch.Tensor:
|
||||
assert inputs_embeds is not None
|
||||
# Fused: zero pos-0 embeds + enorm(embeds) + hnorm(prev) + cat -> [N, 2H].
|
||||
eh_input = fused_eh_norm(
|
||||
positions,
|
||||
inputs_embeds,
|
||||
previous_hidden_states,
|
||||
self.enorm.weight,
|
||||
self.hnorm.weight,
|
||||
self.enorm.variance_epsilon,
|
||||
)
|
||||
# Fused zero pos-0 + enorm(embeds) + hnorm(prev) + cat -> [N, 2H]. With a
|
||||
# replicated table the caller passes ``embed_table`` so the embedding
|
||||
# lookup is folded in too (fused_embed_eh_norm); otherwise the embeds are
|
||||
# precomputed and go through the model-local fused_eh_norm.
|
||||
if embed_table is not None:
|
||||
eh_input = fused_embed_eh_norm(
|
||||
positions,
|
||||
input_ids,
|
||||
embed_table,
|
||||
previous_hidden_states,
|
||||
self.enorm.weight,
|
||||
self.hnorm.weight,
|
||||
self.enorm.variance_epsilon,
|
||||
)
|
||||
else:
|
||||
assert inputs_embeds is not None
|
||||
eh_input = fused_eh_norm(
|
||||
positions,
|
||||
inputs_embeds,
|
||||
previous_hidden_states,
|
||||
self.enorm.weight,
|
||||
self.hnorm.weight,
|
||||
self.enorm.variance_epsilon,
|
||||
)
|
||||
is_sequence_parallel = self.mtp_block.use_sequence_parallel
|
||||
if is_sequence_parallel:
|
||||
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
|
||||
@@ -158,11 +175,15 @@ class DeepseekV32MultiTokenPredictor(nn.Module):
|
||||
)
|
||||
}
|
||||
)
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
self.embed_tokens = make_input_embedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
quant_config=vllm_config.quant_config,
|
||||
prefix=maybe_prefix(prefix, "embed_tokens"),
|
||||
tie_word_embeddings=getattr(config, "tie_word_embeddings", False),
|
||||
)
|
||||
# A full on-rank table lets the eh_norm fusion fold in the embedding gather.
|
||||
self.replicated_embed = has_full_vocab_on_rank(self.embed_tokens)
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size)
|
||||
|
||||
def set_skip_topk(self, skip: bool):
|
||||
@@ -192,14 +213,21 @@ class DeepseekV32MultiTokenPredictor(nn.Module):
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
# With a replicated table, defer the embedding gather to fused_eh_norm
|
||||
# (folded into the enorm/hnorm/cat launch); otherwise gather it here.
|
||||
embed_table = None
|
||||
if inputs_embeds is None:
|
||||
inputs_embeds = self.embed_tokens(input_ids)
|
||||
if self.replicated_embed:
|
||||
embed_table = self.embed_tokens.weight
|
||||
else:
|
||||
inputs_embeds = self.embed_tokens(input_ids)
|
||||
current_step_idx = spec_step_idx % self.num_mtp_layers
|
||||
return self.layers[str(self.mtp_start_layer_idx + current_step_idx)](
|
||||
input_ids,
|
||||
positions,
|
||||
previous_hidden_states,
|
||||
inputs_embeds,
|
||||
embed_table,
|
||||
current_step_idx,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user