# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import typing from collections.abc import Callable, Iterable from itertools import islice import regex as re import torch import torch.nn as nn from vllm.compilation.decorators import support_torch_compile from vllm.config import VllmConfig from vllm.distributed import ( get_pp_group, get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size, ) from vllm.model_executor.layers.activation import SiluAndMul, SiluAndMulWithClamp from vllm.model_executor.layers.fused_moe import ( FusedMoEFactory, GateLinear, fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.fused_moe.utils import ( is_model_fused_shared_expert_compatible, ) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, RowParallelLinear, ) from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.mhc import ( HAS_AITER_MHC, HAS_TILELANG_MHC, HCHeadOp, MHCFusedPostPreOp, MHCPostOp, MHCPreOp, ) from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.interfaces import ( EagleModelMixin, SupportsEagle3, SupportsPP, ) from vllm.model_executor.models.utils import ( AutoWeightsLoader, PPMissingLayer, WeightsMapper, extract_layer_index, is_pp_missing_parameter, make_layers, maybe_prefix, ) from vllm.models.deepseek_v4.cpu.cpu_sparse import DeepseekV4CPUAttention from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors class DeepseekV4MLP(nn.Module): def __init__( self, hidden_size: int, intermediate_size: int, hidden_act: str, swiglu_limit: float | None = None, quant_config: QuantizationConfig | None = None, reduce_results: bool = True, is_sequence_parallel: bool = False, prefix: str = "", ) -> None: super().__init__() # is_sequence_parallel: weights are replicated, no collective ops # needed. Otherwise: standard TP with an allreduce at the end. self.gate_up_proj = MergedColumnParallelLinear( hidden_size, [intermediate_size] * 2, bias=False, quant_config=quant_config, disable_tp=is_sequence_parallel, prefix=f"{prefix}.gate_up_proj", ) self.down_proj = RowParallelLinear( intermediate_size, hidden_size, bias=False, quant_config=quant_config, reduce_results=reduce_results, disable_tp=is_sequence_parallel, prefix=f"{prefix}.down_proj", ) if hidden_act != "silu": raise ValueError( f"Unsupported activation: {hidden_act}. Only silu is supported for now." ) if swiglu_limit is not None: self.act_fn = SiluAndMulWithClamp(swiglu_limit) else: self.act_fn = SiluAndMul() def forward(self, x): gate_up, _ = self.gate_up_proj(x) x = self.act_fn(gate_up) x, _ = self.down_proj(x) return x class DeepseekV4MoE(nn.Module): def __init__( self, vllm_config: VllmConfig, prefix: str = "", ): super().__init__() self.tp_size = get_tensor_model_parallel_world_size() config = vllm_config.model_config.hf_config quant_config = vllm_config.quant_config self.prefix = prefix self.routed_scaling_factor = getattr(config, "routed_scaling_factor", 1.0) self.hidden_size = config.hidden_size self.n_routed_experts = config.n_routed_experts self.n_activated_experts = config.num_experts_per_tok self.moe_intermediate_size = config.moe_intermediate_size self.swiglu_limit = config.swiglu_limit self.renormalize = config.norm_topk_prob self.scoring_func = getattr(config, "scoring_func", "sqrtsoftplus") self.gate = GateLinear( input_size=config.hidden_size, output_size=config.n_routed_experts, bias=False, out_dtype=torch.float32, prefix=f"{prefix}.gate", ) self.gate.e_score_correction_bias = None self.gate.tid2eid = None is_hash_moe = extract_layer_index(prefix) < config.mlp_layer_types.count( "hash_moe" ) self.hash_indices_dtype = torch.int32 if is_hash_moe: # hash MoE doesn't use e_score_correction_bias # Use randint instead of empty to avoid garbage values causing # invalid memory access in dummy mode (--load-format="dummy") self.gate.tid2eid = nn.Parameter( torch.randint( 0, config.n_routed_experts, (config.vocab_size, config.num_experts_per_tok), dtype=self.hash_indices_dtype, ), requires_grad=False, ) elif getattr(config, "topk_method", None) == "noaux_tc": self.gate.e_score_correction_bias = nn.Parameter( torch.empty(config.n_routed_experts, dtype=torch.float32), requires_grad=False, ) self.n_shared_experts = config.n_shared_experts # No CPU fused-shared-expert grouped-GEMM path yet. self.is_fused_shared_expert_enabled = False if config.n_shared_experts is None or self.is_fused_shared_expert_enabled: self.shared_experts = None else: intermediate_size = config.moe_intermediate_size * config.n_shared_experts self.shared_experts = DeepseekV4MLP( hidden_size=config.hidden_size, intermediate_size=intermediate_size, hidden_act=config.hidden_act, swiglu_limit=self.swiglu_limit, quant_config=quant_config, reduce_results=False, prefix=f"{prefix}.shared_experts", ) self.tp_rank = get_tensor_model_parallel_rank() assert config.n_routed_experts % self.tp_size == 0 self.n_local_experts = config.n_routed_experts // self.tp_size self.experts_start_idx = self.tp_rank * self.n_local_experts self.experts_end_idx = self.experts_start_idx + self.n_local_experts self.experts = FusedMoEFactory( shared_experts=self.shared_experts, n_shared_experts=( config.n_shared_experts if self.is_fused_shared_expert_enabled else None ), fuse_shared_experts=self.is_fused_shared_expert_enabled, gate=self.gate, num_experts=config.n_routed_experts, top_k=config.num_experts_per_tok, hidden_size=config.hidden_size, intermediate_size=config.moe_intermediate_size, renormalize=config.norm_topk_prob, quant_config=quant_config, prefix=f"{prefix}.experts", scoring_func=self.scoring_func, routed_scaling_factor=self.routed_scaling_factor, e_score_correction_bias=self.gate.e_score_correction_bias, hash_indices_table=self.gate.tid2eid, swiglu_limit=self.swiglu_limit, router_logits_dtype=torch.float32, ) def forward( self, hidden_states: torch.Tensor, input_ids: torch.Tensor | None = None ) -> torch.Tensor: if self.gate.tid2eid is not None and input_ids is None: raise ValueError("DeepSeek V4 hash MoE routing requires input_ids.") org_shape = hidden_states.shape final_hidden_states = self.experts( hidden_states=hidden_states, router_logits=hidden_states, input_ids=input_ids, ) return final_hidden_states.view(org_shape) class DeepseekV4DecoderLayer(nn.Module): def __init__( self, vllm_config, prefix, topk_indices_buffer: torch.Tensor | None = None, ): super().__init__() # Lazy import to avoid top-level tilelang dependency. # Registers both torch.ops.vllm.mhc_pre and mhc_post import vllm.model_executor.layers.mhc # noqa: F401 config = vllm_config.model_config.hf_config self.hidden_size = config.hidden_size self.rms_norm_eps = config.rms_norm_eps self.attn = DeepseekV4CPUAttention( vllm_config, prefix=f"{prefix}.attn", topk_indices_buffer=topk_indices_buffer, ) self.ffn = DeepseekV4MoE(vllm_config, prefix=f"{prefix}.ffn") self.attn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps) self.ffn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps) self.hc_mult = config.hc_mult self.hc_sinkhorn_iters = config.hc_sinkhorn_iters self.hc_eps = config.hc_eps self.hc_post_alpha = 2.0 mix_hc = (2 + self.hc_mult) * self.hc_mult hc_dim = self.hc_mult * self.hidden_size self.hc_attn_fn = nn.Parameter( torch.empty( (mix_hc, hc_dim), dtype=torch.float32, ), requires_grad=False, ) self.hc_ffn_fn = nn.Parameter( torch.empty( (mix_hc, hc_dim), dtype=torch.float32, ), requires_grad=False, ) self.hc_attn_base = nn.Parameter( torch.empty( mix_hc, dtype=torch.float32, ), requires_grad=False, ) self.hc_ffn_base = nn.Parameter( torch.empty( mix_hc, dtype=torch.float32, ), requires_grad=False, ) self.hc_attn_scale = nn.Parameter( torch.empty( 3, dtype=torch.float32, ), requires_grad=False, ) self.hc_ffn_scale = nn.Parameter( torch.empty( 3, dtype=torch.float32, ), requires_grad=False, ) self.mhc_pre = MHCPreOp() self.mhc_post = MHCPostOp() self.mhc_fused_post_pre = MHCFusedPostPreOp() self.use_fused_mhc = HAS_TILELANG_MHC and not ( HAS_AITER_MHC and self.hidden_size % 256 == 0 ) def hc_pre( self, x: torch.Tensor, hc_fn: torch.Tensor, hc_scale: torch.Tensor, hc_base: torch.Tensor, ): post_mix, res_mix, layer_input = self.mhc_pre( residual=x, fn=hc_fn, hc_scale=hc_scale, hc_base=hc_base, rms_eps=self.rms_norm_eps, hc_pre_eps=self.hc_eps, hc_sinkhorn_eps=self.hc_eps, hc_post_mult_value=self.hc_post_alpha, sinkhorn_repeat=self.hc_sinkhorn_iters, ) return layer_input, post_mix, res_mix def hc_post( self, x: torch.Tensor, residual: torch.Tensor, post: torch.Tensor, comb: torch.Tensor, ): return self.mhc_post(x, residual, post, comb) def _forward_fused_post_pre( self, x: torch.Tensor, positions: torch.Tensor, input_ids: torch.Tensor | None, post_mix: torch.Tensor | None = None, res_mix: torch.Tensor | None = None, residual: torch.Tensor | None = None, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: if residual is None: # Run standalone hc_pre on first layer residual = x x, post_mix, res_mix = self.hc_pre( x, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base ) else: residual, post_mix, res_mix, x = self.mhc_fused_post_pre( x, residual, post_mix, res_mix, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base, self.rms_norm_eps, self.hc_eps, self.hc_eps, self.hc_post_alpha, self.hc_sinkhorn_iters, ) x = self.attn_norm(x) x = self.attn(positions, x, None) residual, post_mix, res_mix, x = self.mhc_fused_post_pre( x, residual, post_mix, res_mix, self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base, self.rms_norm_eps, self.hc_eps, self.hc_eps, self.hc_post_alpha, self.hc_sinkhorn_iters, ) x = self.ffn_norm(x) x = self.ffn(x, input_ids) return x, residual, post_mix, res_mix def _forward_unfused_post_pre( self, x: torch.Tensor, positions: torch.Tensor, input_ids: torch.Tensor | None, post_mix: torch.Tensor | None = None, res_mix: torch.Tensor | None = None, residual: torch.Tensor | None = None, ) -> tuple[ torch.Tensor, torch.Tensor | None, torch.Tensor | None, torch.Tensor | None ]: residual = x x, post, comb = self.hc_pre( x, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base ) x = self.attn_norm(x) x = self.attn(positions, x, None) x = self.hc_post(x, residual, post, comb) residual = x x, post, comb = self.hc_pre( x, self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base ) x = self.ffn_norm(x) x = self.ffn(x, input_ids) x = self.hc_post(x, residual, post, comb) return x, None, None, None def forward( self, x: torch.Tensor, positions: torch.Tensor, input_ids: torch.Tensor | None, post_mix: torch.Tensor | None = None, res_mix: torch.Tensor | None = None, residual: torch.Tensor | None = None, ) -> tuple[ torch.Tensor, torch.Tensor | None, torch.Tensor | None, torch.Tensor | None ]: if not self.use_fused_mhc: return self._forward_unfused_post_pre( x, positions, input_ids, post_mix, res_mix, residual ) return self._forward_fused_post_pre( x, positions, input_ids, post_mix, res_mix, residual ) @support_torch_compile( dynamic_arg_dims={ "input_ids": 0, "positions": -1, "intermediate_tensors": 0, "inputs_embeds": 0, } ) class DeepseekV4Model(nn.Module, EagleModelMixin): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() config = vllm_config.model_config.hf_config quant_config = vllm_config.quant_config self.config = config self.vocab_size = config.vocab_size self.hc_eps = config.hc_eps self.hc_mult = config.hc_mult self.hc_dim = self.hc_mult * config.hidden_size self.rms_norm_eps = config.rms_norm_eps self.device = current_platform.device_type # Reserved topk indices buffer for all Indexer layers to reuse. self.topk_indices_buffer = torch.empty( vllm_config.scheduler_config.max_num_batched_tokens, config.index_topk, dtype=torch.int32, device=self.device, ) if get_pp_group().is_first_rank: self.embed_tokens = VocabParallelEmbedding( config.vocab_size, config.hidden_size, quant_config=quant_config, prefix=f"{prefix}.embed_tokens", ) else: self.embed_tokens = PPMissingLayer() self.start_layer, self.end_layer, self.layers = make_layers( config.num_hidden_layers, lambda prefix: DeepseekV4DecoderLayer( vllm_config, prefix=prefix, topk_indices_buffer=self.topk_indices_buffer, ), prefix=f"{prefix}.layers", ) self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible( self.layers, DeepseekV4MoE, "ffn", ) if get_pp_group().is_last_rank: self.norm = RMSNorm(config.hidden_size, self.rms_norm_eps) else: self.norm = PPMissingLayer() self.hc_head_fn = nn.Parameter( torch.empty( self.hc_mult, self.hc_dim, dtype=torch.float32, ), requires_grad=False, ) self.hc_head_base = nn.Parameter( torch.empty( self.hc_mult, dtype=torch.float32, ), requires_grad=False, ) self.hc_head_scale = nn.Parameter( torch.empty(1, dtype=torch.float32), requires_grad=False, ) self.hc_head_op = HCHeadOp() spec_config = vllm_config.speculative_config needs_mtp_hidden_states = spec_config is not None and ( spec_config.use_eagle() or spec_config.uses_draft_model() ) if get_pp_group().is_last_rank and needs_mtp_hidden_states: self._mtp_hidden_buffer = torch.empty( vllm_config.scheduler_config.max_num_batched_tokens, self.hc_dim, dtype=vllm_config.model_config.dtype, device=self.device, ) else: self._mtp_hidden_buffer = None def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.embed_tokens(input_ids) def make_empty_intermediate_tensors( self, batch_size: int, dtype: torch.dtype, device: torch.device, ) -> IntermediateTensors: # PP intermediate tensors keep the (num_tokens, hc_mult, hidden_size) # multi-stream shape until hc_head() collapses it. return IntermediateTensors( { "hidden_states": torch.zeros( (batch_size, self.hc_mult, self.config.hidden_size), dtype=dtype, device=device, ), } ) def forward( self, input_ids: torch.Tensor, positions: torch.Tensor, intermediate_tensors: IntermediateTensors | None, inputs_embeds: torch.Tensor | None = None, ) -> torch.Tensor | IntermediateTensors: if get_pp_group().is_first_rank: if inputs_embeds is not None: hidden_states = inputs_embeds else: hidden_states = self.embed_input_ids(input_ids) hidden_states = hidden_states.unsqueeze(-2).repeat(1, self.hc_mult, 1) else: assert intermediate_tensors is not None hidden_states = intermediate_tensors["hidden_states"] residual, post_mix, res_mix = None, None, None # EAGLE3/DSpark/DFlash aux hidden states: post-mhc hidden state at # each target layer, averaged over hc_mult streams to [T, # hidden_size]. Empty unless a draft model set aux_hidden_state_layers. aux_hidden_states: list[torch.Tensor] = [] # Reused below to avoid computing hc_post twice if the last layer is # also an aux layer. final_aux_recon: torch.Tensor | None = None for idx, layer in enumerate( islice(self.layers, self.start_layer, self.end_layer), start=self.start_layer, ): hidden_states, residual, post_mix, res_mix = layer( hidden_states, positions, input_ids, post_mix, res_mix, residual, ) if (idx + 1) in self.aux_hidden_state_layers: # Unfused path already applied hc_post; fused path needs it # applied here before averaging. if layer.use_fused_mhc: aux_recon = layer.hc_post( hidden_states, residual, post_mix, res_mix ) final_aux_recon = aux_recon else: aux_recon = hidden_states aux_hidden_states.append(aux_recon.mean(dim=1)) if layer is not None and layer.use_fused_mhc: # Reuse the last layer's hc_post output if already computed above. if ( final_aux_recon is not None and self.end_layer in self.aux_hidden_state_layers ): hidden_states = final_aux_recon else: hidden_states = layer.hc_post( hidden_states, residual, post_mix, res_mix ) if not get_pp_group().is_last_rank: return IntermediateTensors({"hidden_states": hidden_states}) if self._mtp_hidden_buffer is not None: num_tokens = hidden_states.shape[0] self._mtp_hidden_buffer[:num_tokens].copy_(hidden_states.flatten(1)) hidden_states = self.hc_head_op( hidden_states, self.hc_head_fn, self.hc_head_scale, self.hc_head_base, self.rms_norm_eps, self.hc_eps, ) hidden_states = self.norm(hidden_states) if len(aux_hidden_states) > 0: return hidden_states, aux_hidden_states return hidden_states def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: stacked_params_mapping = [ # (param_name, shard_name, shard_id) ("gate_up_proj", "w1", 0), ("gate_up_proj", "w3", 1), ("attn.fused_wqa_wkv", "attn.wq_a", 0), ("attn.fused_wqa_wkv", "attn.wkv", 1), ("compressor.fused_wkv_wgate", "compressor.wkv", 0), ("compressor.fused_wkv_wgate", "compressor.wgate", 1), ] params_dict = dict(self.named_parameters()) loaded_params: set[str] = set() def _resolve_param_name(name: str) -> str: inv_name = f"{name}_inv" if name not in params_dict and inv_name in params_dict: return inv_name return name # Pre-compute expert mapping ONCE. expert_mapping = self.get_expert_mapping() # Each MoE's own per-layer fusion decision is the source of truth, # so the redirect below can't diverge from how it was built. fuse_by_layer = { extract_layer_index(mod_name): mod.is_fused_shared_expert_enabled for mod_name, mod in self.named_modules() if isinstance(mod, DeepseekV4MoE) } n_routed = self.config.n_routed_experts # Maps the single shared-expert tensor group to one appended slot; # multiple shared experts would need per-expert slicing (see # deepseek_v2.py). DeepSeek-V4 has n_shared_experts == 1. if any(fuse_by_layer.values()) and self.config.n_shared_experts != 1: raise NotImplementedError( "deepseek-v4 fused shared-expert loading supports only " f"n_shared_experts == 1, got {self.config.n_shared_experts}" ) for name, loaded_weight in weights: # Shared-expert fusion: redirect shared_experts.w{1,2,3} into # the appended routed-expert slot so it loads through the # routed expert (grouped GEMM) loader. if ".ffn.shared_experts.w" in name and fuse_by_layer.get( extract_layer_index(name), False ): name = name.replace( ".ffn.shared_experts.w", f".ffn.experts.{n_routed}.w", ) for param_name, weight_name, shard_id in stacked_params_mapping: # Skip non-stacked layers and experts (experts handled below). if ".experts." in name: continue if weight_name not in name: continue name = name.replace(weight_name, param_name) if is_pp_missing_parameter(name, self): break param_name = _resolve_param_name(name) param = params_dict[param_name] weight_loader = param.weight_loader weight_loader(param, loaded_weight, shard_id) loaded_params.add(param_name) break else: if ".experts." in name: # E8M0 scales are float8_e8m0fnu in checkpoints but the # MoE param is uint8; copy_() would numerically convert # (e.g. 2^-7 -> 0) instead of preserving raw bytes. if ( "weight_scale" in name and loaded_weight.dtype == torch.float8_e8m0fnu ): loaded_weight = loaded_weight.view(torch.uint8) for mapping in expert_mapping: param_name, weight_name, expert_id, expert_shard_id = mapping if weight_name not in name: continue name_mapped = name.replace(weight_name, param_name) if is_pp_missing_parameter(name_mapped, self): continue param = params_dict[name_mapped] # return_success lets us skip experts that have # another replica available. weight_loader = typing.cast( Callable[..., bool], param.weight_loader ) success = weight_loader( param, loaded_weight, name_mapped, shard_id=expert_shard_id, expert_id=expert_id, return_success=True, ) if success: name = name_mapped break loaded_params.add(name_mapped) continue else: if is_pp_missing_parameter(name, self): continue param_name = _resolve_param_name(name) param = params_dict[param_name] weight_loader = getattr( param, "weight_loader", default_weight_loader ) weight_loader(param, loaded_weight) loaded_params.add(param_name) continue return loaded_params def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: # (param_name, weight_name, expert_id, shard_id). When fusing shared # experts, include the appended slots so redirected shared-expert # weights route through the expert loader. n_shared = getattr(self.config, "n_shared_experts", 0) or 0 num_experts = self.config.n_routed_experts + ( n_shared if self.is_fused_shared_expert_enabled else 0 ) return fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="w1", ckpt_down_proj_name="w2", ckpt_up_proj_name="w3", num_experts=num_experts, ) def _make_deepseek_v4_weights_mapper( expert_dtype: str, fuse_shared_experts: bool = False ) -> WeightsMapper: if expert_dtype == "fp4": # MXFP4 experts register scales as ``w{1,2,3}_weight_scale`` (no # _inv); FP8 linear/shared-expert scales register as # ``weight_scale_inv``. Route DeepSeek-native ``.scale`` and # AMD-Quark ``.weight_scale`` keys accordingly, leaving expert/ # shared-expert ``w{1,2,3}.weight_scale`` untouched for the MXFP4 # loader. scale_regex = { re.compile(r"(\.experts\.\d+\.w[123])\.scale$"): r"\1.weight_scale", re.compile(r"\.scale$"): ".weight_scale_inv", re.compile(r"(? torch.Tensor: return self.model.embed_input_ids(input_ids) def compute_logits( self, hidden_states: torch.Tensor, ) -> torch.Tensor | None: logits = self.logits_processor(self.lm_head, hidden_states) return logits def forward( self, input_ids: torch.Tensor, positions: torch.Tensor, intermediate_tensors: IntermediateTensors | None = None, inputs_embeds: torch.Tensor | None = None, ) -> torch.Tensor | IntermediateTensors: hidden_states = self.model( input_ids, positions, intermediate_tensors, inputs_embeds ) return hidden_states def get_mtp_target_hidden_states(self) -> torch.Tensor | None: """Pre-hc_head residual stream buffer (max_num_batched_tokens, hc_mult * hidden_size) for the MTP draft model. Populated by forward(); valid after each target step.""" return getattr(self.model, "_mtp_hidden_buffer", None) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: loader = AutoWeightsLoader(self) return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) def process_weights_after_loading(self) -> None: # No CPU AMX weight-packing hook yet. pass def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: return self.model.get_expert_mapping()