# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import typing from collections.abc import Callable, Iterable from functools import partial from itertools import islice import regex as re import torch import torch.nn as nn import vllm.envs as envs from vllm.config import VllmConfig from vllm.config.kernel import MEGA_MOE_BACKENDS, NATIVE_MEGA_MOE_BACKENDS from vllm.distributed import ( get_engram_dp_size, get_pcp_group, get_pp_group, get_tensor_model_parallel_world_size, get_tp_group, tensor_model_parallel_all_reduce, ) from vllm.forward_context import ( get_forward_context, in_piecewise_cudagraph, is_forward_context_available, ) from vllm.logger import init_logger from vllm.model_executor.kernels.mhc.tilelang import ( mhc_post_tilelang, mhc_pre_delayed_tilelang, ) from vllm.model_executor.kernels.mhc.triton import hc_collapse_triton from vllm.model_executor.layers.fused_moe import ( fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe import ( TrtLlmMxfp4ExpertsModular, ) from vllm.model_executor.layers.fused_moe.moe_output import ( MoEOutput, UnfinalizedMoEOutput, ) from vllm.model_executor.layers.fused_moe.runner.moe_runner import _unpack from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) from vllm.model_executor.model_loader.weight_utils import ( default_weight_loader, drop_checkpoint_cache, ) from vllm.model_executor.models.interfaces import ( EagleModelMixin, MixtureOfExperts, SupportsEagle3, SupportsLoRA, SupportsPP, ) from vllm.model_executor.models.utils import ( AutoWeightsLoader, PPMissingLayer, WeightsMapper, extract_layer_index, is_pp_missing_parameter, make_layers, maybe_prefix, ) from vllm.models.common.ops.sequence_parallel import ( sp_all_gather, sp_padding_mask, sp_reduce_scatter, sp_shard, ) from vllm.models.deepseek_v4.nvidia.model import ( DeepseekV4MoE as DeepseekV4MoEBase, ) from vllm.models.deepseek_v4.nvidia.model import ( MegaGateRoutingMetadata, make_deepseek_v4_expert_params_mapping, prepare_mega_gate_routing_metadata, ) from vllm.models.deepseek_v41.attention import DeepseekV4Attention from vllm.models.deepseek_v41.decoder_replay_layers import DecoderReplayLayers from vllm.models.deepseek_v41.nvidia.flash_mla_mega_attn import ( DeepseekV4MegaAttnAttention, ) from vllm.models.deepseek_v41.nvidia.flashinfer_sparse import ( DeepseekV4FlashInferMLAAttention, DeepseekV4FlashInferSM120Attention, ) from vllm.models.deepseek_v41.nvidia.flashmla import DeepseekV4FlashMLAAttention from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors from vllm.utils.math_utils import cdiv from vllm.v1.attention.backends.registry import AttentionBackendEnum from vllm.v1.worker.ubatching import dbo_current_ubatch_id from ..common.engram import ( Engram, EngramBatch, EngramLayout, NgramHashState, can_share_engram_tables, gather_engram_hashes, ) from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID, image_sentinel_mask from .ops.mhc import ( MHC_OVERLAP_MAX_TOKENS, init_mhc_all_reduce, mhc_pre_delayed_overlap, mhc_shifted_post_pre, supports_mhc_all_reduce, supports_mhc_overlap, ) if typing.TYPE_CHECKING: from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata logger = init_logger(__name__) class DeepseekV4MoE(DeepseekV4MoEBase): def __init__( self, vllm_config: VllmConfig, prefix: str = "", use_sequence_parallel: bool = False, reduce_results: bool = True, ): config = vllm_config.model_config.hf_config n_routed_experts = config.n_routed_experts n_activated_experts = config.num_experts_per_tok if extract_layer_index(prefix) >= config.num_hidden_layers: n_routed_experts = ( getattr(config, "dspark_n_routed_experts", 0) or n_routed_experts ) n_activated_experts = ( getattr(config, "dspark_num_experts_per_tok", 0) or n_activated_experts ) super().__init__( vllm_config, prefix=prefix, use_sequence_parallel=use_sequence_parallel, n_routed_experts=n_routed_experts, n_activated_experts=n_activated_experts, num_hash_layers=0, reduce_results=reduce_results, image_sentinel_lo=IMAGE_SENTINEL_BASE_ID, ) def defer_finalize(self) -> None: """Leave the routed top-k reduction to the next fused all-reduce + mHC. Only the modular TRTLLM MXFP4 experts can stop after GEMM2; the fused kernel takes as many tokens as the mHC overlap path. """ moe_config = self.experts.moe_config experts_cls = getattr( self.experts.routed_experts.quant_method, "experts_cls", None ) if ( experts_cls is TrtLlmMxfp4ExpertsModular and self.experts.routed_scaling_factor == 1.0 ): moe_config.defer_moe_finalize(MHC_OVERLAP_MAX_TOKENS) if moe_config.use_deferred_moe_finalize: logger.info_once( "DSV4.1 mHC: MoE top-k finalize fused into the all-reduce for " "up to %d tokens.", MHC_OVERLAP_MAX_TOKENS, ) def defers_finalize(self, num_tokens: int) -> bool: return ( not self.use_native_mega_moe and self.experts.moe_config.should_defer_moe_finalize(num_tokens) ) def forward_unfinalized( self, hidden_states: torch.Tensor, input_ids: torch.Tensor | None ) -> MoEOutput: """``forward`` with the routed top-k reduction and all-reduce left open.""" # The runner's custom op returns tensors only, so run its body directly. shared_output, routed = _unpack( self.experts._forward_impl( hidden_states, hidden_states, hidden_states, input_ids ) ) assert shared_output is not None assert isinstance(routed, UnfinalizedMoEOutput) return MoEOutput(routed=routed, shared_output=shared_output) def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]: """Pick the CUDA sparse-MLA attention class for the configured backend. The generic CUDA backend selector does not instantiate DSv4 layers directly, so map generic sparse-MLA choices to the DSv4-specialized attention class. Without an explicit backend: SM12 takes FlashInfer, SM100 takes mega attention where the topology allows it, and everything else keeps the FlashMLA path. """ backend = vllm_config.attention_config.backend if vllm_config.parallel_config.prefill_context_parallel_size > 1: if backend not in ( None, AttentionBackendEnum.FLASHMLA_SPARSE, AttentionBackendEnum.FLASHMLA_SPARSE_DSV4, AttentionBackendEnum.FLASHMLA_SPARSE_DSV41, ): raise NotImplementedError( "DeepSeek-V4.1 PCP requires FlashMLA sparse attention." ) return DeepseekV4FlashMLAAttention device_capability = current_platform.get_device_capability() if backend in ( AttentionBackendEnum.FLASHINFER_MLA_SPARSE, AttentionBackendEnum.FLASHINFER_MLA_SPARSE_SM120, ): raise ValueError( f"{backend.name} is not a DeepSeek V4.1 attention backend. " "Use FLASHINFER_MLA_SPARSE_DSV41 for DeepSeek V4.1 FlashInfer " "sparse MLA." ) if backend in ( AttentionBackendEnum.FLASHINFER_MLA_SPARSE_DSV4, AttentionBackendEnum.FLASHINFER_MLA_SPARSE_DSV41, ): if device_capability is not None and device_capability.major == 12: return DeepseekV4FlashInferSM120Attention return DeepseekV4FlashInferMLAAttention if backend is AttentionBackendEnum.FLASHMLA_MEGA_ATTN_DSV41: return DeepseekV4MegaAttnAttention if backend in ( AttentionBackendEnum.FLASHMLA_SPARSE, AttentionBackendEnum.FLASHMLA_SPARSE_DSV4, AttentionBackendEnum.FLASHMLA_SPARSE_DSV41, ): return DeepseekV4FlashMLAAttention if device_capability is not None and device_capability.major == 12: return DeepseekV4FlashInferSM120Attention # Mega attention is the SM100 default: it fuses Q RoPE, sparse attention, # the output's inverse RoPE and its FP8 cast into one launch, and brings # the 288 B NVFP4 compressed record -- the format the reference # implementation itself stores. It declines topologies it cannot serve # (non-SM100, TP that leaves fewer than WV_GROUP_SIZE heads per wo_a # group, a build without the kernel), which then fall through to FlashMLA. if DeepseekV4MegaAttnAttention.is_available_for(vllm_config): return DeepseekV4MegaAttnAttention return DeepseekV4FlashMLAAttention def _use_sequence_parallel(vllm_config: VllmConfig) -> bool: parallel_config = vllm_config.parallel_config moe_needs_token_sharded_input = ( vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS ) return ( parallel_config.pipeline_parallel_size == 1 and parallel_config.enable_expert_parallel and parallel_config.tensor_parallel_size > 1 and (moe_needs_token_sharded_input or parallel_config.data_parallel_size > 1) ) def _shard_fused_wqa_wkv(vllm_config: VllmConfig) -> bool: """Whether large batches shard fused_wqa_wkv by token and all-gather the output.""" ca_comm = getattr(get_tp_group().device_communicator, "ca_comm", None) return ( current_platform.is_device_capability(90) and not _use_sequence_parallel(vllm_config) and vllm_config.lora_config is None and not vllm_config.parallel_config.use_ubatching and not envs.VLLM_BATCH_INVARIANT and ca_comm is not None and not ca_comm.disabled and ca_comm.fully_connected ) def maybe_init_gemm_rs(vllm_config: VllmConfig, use_sequence_parallel: bool) -> bool: """Set up the fused ``wo_b`` GEMM + reduce-scatter when opted in. Sequence parallel is the only topology where ``wo_b`` ends in a reduce-scatter, so the kernel is bound to RS mode. The workspace is a process-wide singleton shared with any other model code in this worker (the DSpark drafter reuses it), and its NVLink multicast rendezvous is collective, so every TP rank must take the same decision here. """ if not (use_sequence_parallel and envs.VLLM_ENABLE_GEMM_RS): return False # The kernel module pulls in cute_dsl, so import it only once opted in. from vllm.model_executor.kernels.linear.cute_dsl.gemm_rs_ar import ( maybe_init_gemm_rs_ar, ) hidden_size = vllm_config.model_config.hf_config.hidden_size if not maybe_init_gemm_rs_ar(vllm_config, N=hidden_size, all_reduce=False): return False logger.info_once("To disable DeepSeek-V4.1 GEMM-RS, set VLLM_ENABLE_GEMM_RS=0.") return True class DeepseekV4DecoderLayer(nn.Module): def __init__( self, vllm_config, prefix, topk_indices_buffer: torch.Tensor | None = None, aux_stream_list: list[torch.cuda.Stream] | None = None, candidate_block_buffer: torch.Tensor | None = None, engram_layout: EngramLayout | None = None, engram_prefetch_stream: torch.cuda.Stream | None = None, run_gemm_rs: bool = False, mhc_stream: torch.cuda.Stream | None = None, fuse_mhc_all_reduce: bool = False, ): super().__init__() config = vllm_config.model_config.hf_config self.hidden_size = config.hidden_size self.mhc_stream = mhc_stream self.fuse_mhc_all_reduce = fuse_mhc_all_reduce self.use_sequence_parallel = _use_sequence_parallel(vllm_config) self.engram: Engram | None = None if engram_layout is not None: layer_id = extract_layer_index(prefix) if layer_id in engram_layout.layer_ids: self.engram = Engram( config, vllm_config.quant_config, engram_layout, engram_layout.layer_ids.index(layer_id), use_sequence_parallel=self.use_sequence_parallel, prefix=f"{prefix}.engram", prefetch_stream=engram_prefetch_stream, ) self.rms_norm_eps = config.rms_norm_eps self.attn = _select_dsv4_attn_cls(vllm_config)( vllm_config, prefix=f"{prefix}.attn", topk_indices_buffer=topk_indices_buffer, aux_stream_list=aux_stream_list, candidate_block_buffer=candidate_block_buffer, ) if self.use_sequence_parallel or fuse_mhc_all_reduce: self.attn.wo_b.reduce_results = False if run_gemm_rs: # Binds only when wo_b's kernel and shape qualify; otherwise # forward keeps the separate reduce-scatter below. self.attn.bind_gemm_rs() self.attn.shard_fused_wqa_wkv = _shard_fused_wqa_wkv(vllm_config) self.ffn = DeepseekV4MoE( vllm_config, prefix=f"{prefix}.ffn", use_sequence_parallel=self.use_sequence_parallel, reduce_results=not fuse_mhc_all_reduce, ) self.attn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps) self.ffn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps) self.hc_mult = config.hc_mult self.hc_sinkhorn_iters = config.hc_sinkhorn_iters self.hc_eps = config.hc_eps self.hc_post_alpha = 2.0 if vllm_config.kernel_config.enable_jit_warmup and current_platform.is_cuda(): from vllm.model_executor.kernels.mhc.tilelang_kernels import ( mhc_fused_post_pre_splits, ) from vllm.model_executor.kernels.mhc.warmup import MHC_PRE_NORM_KERNEL max_tokens = vllm_config.scheduler_config.max_num_batched_tokens if self.use_sequence_parallel: tp_size = vllm_config.parallel_config.tensor_parallel_size max_tokens = (max_tokens + tp_size - 1) // tp_size # The epilogue compiles per projection width and per pre-mix mode. # The first layer projects the broadcast embedding, so it reads one # hidden_size-wide row and selects stream zero; every later sublayer # projects the full hc stream, and collapses it with the pre-mix the # previous sublayer carried in. Those later shapes also arrive # through the fused post + pre-norm GEMM, which picks split-k # factors the token sweep below never produces. broadcast_embedding = { "rms_numel": self.hidden_size, "use_pre_mix_in": False, "extra_splits": (), } hc_stream = { "rms_numel": self.hc_mult * self.hidden_size, "extra_splits": mhc_fused_post_pre_splits( self.hidden_size, self.hc_mult ), } variants = [ broadcast_embedding, {**hc_stream, "use_pre_mix_in": False}, {**hc_stream, "use_pre_mix_in": True}, ] if vllm_config.speculative_config is not None: # Draft setups read aux hidden states out of the same collapse, # which is its own specialization of the kernel. variants.append( {**hc_stream, "use_pre_mix_in": True, "write_aux": True} ) # Shifted mHC splits the epilogue in two: the input collapse on the # caller stream and coefficient generation on a side stream. split_modes = ( ("fused", "stats", "input") if mhc_stream is not None else ("fused",) ) for variant in variants: for split_mode in split_modes: MHC_PRE_NORM_KERNEL.register_warmup( split_mode=split_mode, max_tokens=( max_tokens if split_mode == "fused" else min(max_tokens, MHC_OVERLAP_MAX_TOKENS) ), hidden_size=self.hidden_size, rms_eps=self.rms_norm_eps, hc_pre_eps=self.hc_eps, hc_sinkhorn_eps=self.hc_eps, hc_post_mult_value=self.hc_post_alpha, sinkhorn_repeat=self.hc_sinkhorn_iters, norm_eps=self.rms_norm_eps, hc_mult=self.hc_mult, **variant, ) mix_hc = (2 + self.hc_mult) * self.hc_mult hc_dim = self.hc_mult * self.hidden_size self.hc_attn_fn = nn.Parameter( torch.empty( (mix_hc, hc_dim), dtype=torch.float32, ), requires_grad=False, ) self.hc_attn_fn_broadcast: torch.Tensor | None = None self.hc_ffn_fn = nn.Parameter( torch.empty( (mix_hc, hc_dim), dtype=torch.float32, ), requires_grad=False, ) self.hc_attn_base = nn.Parameter( torch.empty( mix_hc, dtype=torch.float32, ), requires_grad=False, ) self.hc_ffn_base = nn.Parameter( torch.empty( mix_hc, dtype=torch.float32, ), requires_grad=False, ) self.hc_attn_scale = nn.Parameter( torch.empty( 3, dtype=torch.float32, ), requires_grad=False, ) self.hc_ffn_scale = nn.Parameter( torch.empty( 3, dtype=torch.float32, ), requires_grad=False, ) def forward( self, x: torch.Tensor | MoEOutput, positions: torch.Tensor, input_ids: torch.Tensor | None, pre_mix: torch.Tensor | None = None, post_mix: torch.Tensor | None = None, res_mix: torch.Tensor | None = None, residual: torch.Tensor | None = None, engram_hashes: torch.Tensor | None = None, engram_mask: torch.Tensor | None = None, *, capture_previous_aux: bool = False, mega_gate_metadata: MegaGateRoutingMetadata | None = None, ) -> tuple[ torch.Tensor | MoEOutput, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor | None, ]: previous_aux: torch.Tensor | None = None mhc_stream = self.mhc_stream if mhc_stream is not None and ( in_piecewise_cudagraph() or not 0 < positions.shape[0] <= MHC_OVERLAP_MAX_TOKENS or not torch.cuda.is_current_stream_capturing() ): # Use overlap only in FULL graphs; eager warmup initializes Mega mHC. mhc_stream = None mhc_pre = ( partial(mhc_pre_delayed_overlap, stream=mhc_stream) if mhc_stream is not None else mhc_pre_delayed_tilelang ) # The reference collapses each sublayer's input with the *previous* # sublayer's pre-mix: attention uses the pre-mix carried in (identity # for the first layer), the FFN uses this layer's attention pre-mix. if residual is None: assert isinstance(x, torch.Tensor) if x.dim() == 2: # First layer: the stream is the embedding broadcast to hc # copies and the identity pre-mix selects copy 0. assert self.hc_attn_fn_broadcast is not None residual = x.unsqueeze(1).expand(-1, self.hc_mult, -1).contiguous() post_mix, res_mix, x, attn_pre = mhc_pre( residual, self.hc_attn_fn_broadcast, self.hc_attn_scale, self.hc_attn_base, self.rms_norm_eps, self.hc_eps, self.hc_eps, self.hc_post_alpha, self.hc_sinkhorn_iters, x=x, norm_weight=self.attn_norm.weight, norm_eps=self.attn_norm.variance_epsilon, ) else: residual = x post_mix, res_mix, x, attn_pre = mhc_pre( residual, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base, self.rms_norm_eps, self.hc_eps, self.hc_eps, self.hc_post_alpha, self.hc_sinkhorn_iters, pre_mix=pre_mix, norm_weight=self.attn_norm.weight, norm_eps=self.attn_norm.variance_epsilon, ) elif self.engram is not None and engram_hashes is not None: # Engram injection happens between the previous sublayer's post # and this block's pre, on the full hc stream, so the mix # coefficients see the injected stream. The injection also keeps # the post out of the pre-norm GEMM's fused prologue. The layer # before an engram layer finalizes its own MoE. assert isinstance(x, torch.Tensor) if self.fuse_mhc_all_reduce: x = tensor_model_parallel_all_reduce(x) previous_post = mhc_post_tilelang(x, residual, post_mix, res_mix) if capture_previous_aux: previous_aux = previous_post.mean(dim=1) residual = self.engram( previous_post, engram_hashes[:, self.engram.layer_hash_index], engram_mask, ) post_mix, res_mix, x, attn_pre = mhc_pre( residual, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base, self.rms_norm_eps, self.hc_eps, self.hc_eps, self.hc_post_alpha, self.hc_sinkhorn_iters, pre_mix=pre_mix, norm_weight=self.attn_norm.weight, norm_eps=self.attn_norm.variance_epsilon, ) else: # The collapse already reads the post-mapped streams, so the mean # aux consumers want comes out of the same kernel. residual, post_mix, res_mix, x, attn_pre, aux = mhc_shifted_post_pre( x, residual, post_mix, res_mix, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base, self.rms_norm_eps, self.hc_eps, self.hc_eps, self.hc_post_alpha, self.hc_sinkhorn_iters, pre_mix=pre_mix, norm_weight=self.attn_norm.weight, norm_eps=self.attn_norm.variance_epsilon, capture_aux=capture_previous_aux, stream=mhc_stream, reduce_results=self.fuse_mhc_all_reduce, ) if capture_previous_aux: previous_aux = aux if self.use_sequence_parallel: x = sp_all_gather(x)[: positions.shape[0]] x = self.attn(positions, x, None) # With GEMM-RS bound, the attention output is already this rank's # sequence-parallel shard (see DeepseekV4Attention._wo_b_proj). if self.use_sequence_parallel and self.attn.gemm_rs is None: x = sp_reduce_scatter(x) if mhc_stream is not None: torch.cuda.current_stream().wait_stream(mhc_stream) residual, post_mix, res_mix, x, ffn_pre, _ = mhc_shifted_post_pre( x, residual, post_mix, res_mix, self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base, self.rms_norm_eps, self.hc_eps, self.hc_eps, self.hc_post_alpha, self.hc_sinkhorn_iters, pre_mix=attn_pre, norm_weight=self.ffn_norm.weight, norm_eps=self.ffn_norm.variance_epsilon, stream=mhc_stream, reduce_results=self.fuse_mhc_all_reduce, ) if self.fuse_mhc_all_reduce and self.ffn.defers_finalize(x.shape[0]): # The next layer's first boundary finalizes it with the all-reduce. x = self.ffn.forward_unfinalized(x, input_ids) else: x = self.ffn(x, input_ids, mega_gate_metadata) if mhc_stream is not None: torch.cuda.current_stream().wait_stream(mhc_stream) return x, residual, post_mix, res_mix, ffn_pre, previous_aux def write_kv( self, x: torch.Tensor | MoEOutput, positions: torch.Tensor, input_ids: torch.Tensor | None, pre_mix: torch.Tensor, post_mix: torch.Tensor, res_mix: torch.Tensor, residual: torch.Tensor, ) -> None: """Write the KV ``forward`` would for the replay layers' inputs, skipping the rest of the layer; the replay batch then reruns its rows.""" # The replay batch rules out Engram and sequence parallel here. assert self.engram is None and not self.use_sequence_parallel *_, x, _, _ = mhc_shifted_post_pre( x, residual, post_mix, res_mix, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base, self.rms_norm_eps, self.hc_eps, self.hc_eps, self.hc_post_alpha, self.hc_sinkhorn_iters, pre_mix=pre_mix, norm_weight=self.attn_norm.weight, norm_eps=self.attn_norm.variance_epsilon, reduce_results=self.fuse_mhc_all_reduce, ) self.attn.forward_kv(positions, x) class DeepseekV4Model(nn.Module, EagleModelMixin): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() config = vllm_config.model_config.hf_config quant_config = vllm_config.quant_config self.config = config self.quant_config = quant_config self.parallel_config = vllm_config.parallel_config if self.parallel_config.prefill_context_parallel_size > 1 and ( self.parallel_config.pipeline_parallel_size != 1 or self.parallel_config.data_parallel_size != 1 or self.parallel_config.decode_context_parallel_size != 1 or self.parallel_config.use_ubatching or vllm_config.speculative_config is not None or vllm_config.kv_transfer_config is not None or vllm_config.cache_config.enable_prefix_caching or vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS ): raise NotImplementedError( "DeepSeek-V4.1 PCP currently requires PP=DP=DCP=1, with no " "microbatching, speculative decoding, KV transfer, prefix " "caching, or MegaMoE." ) self.use_native_mega_moe = ( vllm_config.kernel_config.moe_backend in NATIVE_MEGA_MOE_BACKENDS ) self.use_sequence_parallel = _use_sequence_parallel(vllm_config) if ( self.use_native_mega_moe and not vllm_config.parallel_config.enable_expert_parallel ): raise NotImplementedError( "DeepSeek V4 MegaMoE currently requires expert parallel. " "Enable it with --enable-expert-parallel, or pick a different " "moe backend." ) self.vocab_size = config.vocab_size self.hc_eps = config.hc_eps self.hc_mult = config.hc_mult self.hc_dim = self.hc_mult * config.hidden_size self.rms_norm_eps = config.rms_norm_eps # Three aux streams: one per non-default input GEMM in # DeepseekV4Attention._run_parallel_input_projections # (compressor kv_score, indexer.weights_proj). fused_wqa_wkv stays on # the default stream. aux_stream_list = [torch.cuda.Stream() for _ in range(3)] # Keep mHC independent of the streams used inside attention. mhc_stream = torch.cuda.Stream() if supports_mhc_overlap(vllm_config) else None self.fuse_mhc_all_reduce = mhc_stream is not None and supports_mhc_all_reduce( vllm_config ) if self.fuse_mhc_all_reduce: init_mhc_all_reduce(vllm_config) # Reserved topk indices buffer for all Indexer layers to reuse. self.topk_indices_buffer = torch.empty( vllm_config.scheduler_config.max_num_batched_tokens, config.index_topk, dtype=torch.int32, ) # Two-level candidate filtering: the indexer at # candidate_source_layer_id publishes the top candidate blocks of # compressed positions here; later ratio-1 indexers (24/28/32/36) # mask their scores with it. candidate_source_layer = getattr(config, "candidate_source_layer_id", -1) candidate_topk_blocks = getattr(config, "candidate_topk_blocks", 0) if candidate_source_layer >= 0 and candidate_topk_blocks > 0: self.candidate_block_buffer = torch.empty( vllm_config.scheduler_config.max_num_batched_tokens, candidate_topk_blocks, dtype=torch.int32, ) else: self.candidate_block_buffer = None if get_pp_group().is_first_rank: self.embed_tokens = VocabParallelEmbedding( config.vocab_size, config.hidden_size, quant_config=quant_config, prefix=f"{prefix}.embed_tokens", ) else: self.embed_tokens = PPMissingLayer() self.engram_layout = EngramLayout.from_config(config) engram_config = vllm_config.engram_config self.vllm_config = vllm_config engram_prefetch_stream = ( torch.cuda.Stream() if self.engram_layout is not None and engram_config is not None and engram_config.cpu_offload else None ) if ( self.engram_layout is not None and engram_config is not None and engram_config.dp_shared_memory ): engram_config.dp_shared_memory = can_share_engram_tables(self.engram_layout) if self.engram_layout is not None and engram_config and engram_config.use_thp: # Release old checkpoint cache before allocating the Engram host tables. model_config = vllm_config.model_config drop_checkpoint_cache( model_config.model_weights or model_config.model, revision=model_config.revision, cache_dir=vllm_config.load_config.download_dir, ) # GEMM-RS uses NCCL symmetric-memory multicast, which requires all TP # ranks to belong to one NVLink domain. Collective: run before layers. self.run_gemm_rs = maybe_init_gemm_rs(vllm_config, self.use_sequence_parallel) self.start_layer, self.end_layer, self.layers = make_layers( config.num_hidden_layers, lambda prefix: DeepseekV4DecoderLayer( vllm_config, prefix=prefix, topk_indices_buffer=self.topk_indices_buffer, aux_stream_list=aux_stream_list, candidate_block_buffer=self.candidate_block_buffer, engram_layout=self.engram_layout, engram_prefetch_stream=engram_prefetch_stream, run_gemm_rs=self.run_gemm_rs, mhc_stream=mhc_stream, fuse_mhc_all_reduce=self.fuse_mhc_all_reduce, ), prefix=f"{prefix}.layers", ) self.engram_batch = EngramBatch.create( [ layer.engram for layer in islice(self.layers, self.start_layer, self.end_layer) if getattr(layer, "engram", None) is not None ], vllm_config.scheduler_config.max_num_batched_tokens, ) if self.fuse_mhc_all_reduce: # A MoE's top-k finalize folds into the next layer's first mHC # boundary, which the last local layer lacks and an engram layer # replaces with its own all-reduce. local_layers = list(islice(self.layers, self.start_layer, self.end_layer)) for layer, successor in zip(local_layers, local_layers[1:]): assert isinstance(layer, DeepseekV4DecoderLayer) assert isinstance(successor, DeepseekV4DecoderLayer) if successor.engram is None: layer.ffn.defer_finalize() # Decoder-side SWA bounded replay: in eager prefill steps the layers past # the last KV source run on each request's trailing window only # (decoder_replay_layers.py). self.decoder_replay_layers: DecoderReplayLayers | None = None self.decoder_replay_start = self.end_layer cut = max(config.kv_source_layer_ids) if ( cut < self.end_layer - 1 and self._decoder_replay_supported(vllm_config, cut) and self.layers[cut].attn.swa_cache_layer.bounded_replay ): assert self.layers[cut].attn.compress_ratio <= 1 # The last KV source writes every row's KV, then replays. self.decoder_replay_start = cut # The attention metadata keys the replay layers read. metadata_prefixes = set() for layer in islice(self.layers, cut, self.end_layer): attn = typing.cast(DeepseekV4DecoderLayer, layer).attn metadata_prefixes.add(attn.swa_cache_layer.prefix) if attn.compressed_cache_prefix is not None: metadata_prefixes.add(attn.compressed_cache_prefix) if attn.indexer is not None: metadata_prefixes.add(attn.indexer.k_cache.prefix) self.decoder_replay_layers = DecoderReplayLayers( config.sliding_window, self._run_replay_layers, self.layers[cut].write_kv, metadata_prefixes, self.layers[cut].attn.swa_cache_layer.prefix, ) logger.info_once( "Decoder SWA bounded replay: in eager prefill steps, layer %d " "writes its KV for every token, and the rest of it and layers " "%d-%d run on each request's last %d tokens only.", cut, cut + 1, self.end_layer - 1, config.sliding_window, ) # The n-gram hash needs a slot-keyed rolling store of compressed ids # (chunked prefill / decode lookback); key it off the first local # layer's sliding-window KV cache. Only PP ranks owning an engram # layer need it. self.engram_hash: NgramHashState | None = None self.engram_dp_shared_memory = bool( vllm_config.engram_config and vllm_config.engram_config.dp_shared_memory ) self.engram_swa_prefix: str | None = None if self.engram_layout is not None: local_engram = any( isinstance(layer, DeepseekV4DecoderLayer) and layer.engram is not None for layer in islice(self.layers, self.start_layer, self.end_layer) ) if local_engram: first_layer = next( iter(islice(self.layers, self.start_layer, self.end_layer)) ) swa_cache_module = first_layer.attn.swa_cache_layer self.engram_hash = NgramHashState( vllm_config, self.engram_layout, swa_cache_module ) self.engram_swa_prefix = swa_cache_module.prefix if get_pp_group().is_last_rank: self.norm = RMSNorm(config.hidden_size, self.rms_norm_eps) else: self.norm = PPMissingLayer() spec_config = vllm_config.speculative_config needs_mtp_hidden_states = spec_config is not None and ( spec_config.use_eagle() or spec_config.uses_draft_model() ) if get_pp_group().is_last_rank and needs_mtp_hidden_states: self._mtp_hidden_buffer = torch.empty( vllm_config.scheduler_config.max_num_batched_tokens, self.hc_dim, dtype=vllm_config.model_config.dtype, ) else: self._mtp_hidden_buffer = None def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.embed_tokens(input_ids) def make_empty_intermediate_tensors( self, batch_size: int, dtype: torch.dtype, device: torch.device, ) -> IntermediateTensors: # PP intermediate tensors carry the multi-stream hidden_states # of shape (num_tokens, hc_mult, hidden_size) — V4 expands the # token embedding to hc_mult streams before the first decoder # layer and keeps that shape until the final hc collapse — plus the # (num_tokens, hc_mult) pre-mix the next rank's first layer needs # for its attention collapse. return IntermediateTensors( { "hidden_states": torch.zeros( (batch_size, self.hc_mult, self.config.hidden_size), dtype=dtype, device=device, ), "pre_mix": torch.zeros( (batch_size, self.hc_mult), dtype=torch.float32, device=device, ), } ) def forward( self, input_ids: torch.Tensor, positions: torch.Tensor, intermediate_tensors: IntermediateTensors | None, inputs_embeds: torch.Tensor | None = None, lookback_token_ids: torch.Tensor | None = None, ) -> torch.Tensor | IntermediateTensors: if get_pp_group().is_first_rank: if inputs_embeds is not None: hidden_states = inputs_embeds else: hidden_states = self.embed_input_ids(input_ids) else: assert intermediate_tensors is not None hidden_states = intermediate_tensors["hidden_states"] if self.use_native_mega_moe: input_ids = input_ids.to(torch.int64) # Engram n-gram hashes for the whole (flattened) batch, computed once # on the full token stream — before any sequence-parallel sharding — # and consumed by the engram layers (1 and 14) below. Skipped on # profile runs (KV cache unbound). engram_hashes: torch.Tensor | None = None engram_mask: torch.Tensor | None = None if ( self.engram_hash is not None and input_ids is not None and is_forward_context_available() ): attn_metadata = get_forward_context().attn_metadata if isinstance(attn_metadata, list): attn_metadata = attn_metadata[dbo_current_ubatch_id()] if isinstance(attn_metadata, dict) and self.engram_hash.ensure_cache(): assert self.engram_swa_prefix is not None swa_metadata = typing.cast( "DeepseekSparseSWAMetadata", attn_metadata[self.engram_swa_prefix] ) # Image-span tokens are dead: they break n-grams (hash op # takes True=dead) and their gate is zeroed (Engram.forward # takes True=keep). image_mask = image_sentinel_mask(input_ids) engram_mask = ~image_mask if lookback_token_ids is None: if not self.engram_hash.use_slot_cache: raise NotImplementedError( "engram needs `lookback_token_ids` from the model " "runner (the DBO/ubatch wrapper drops model kwargs)" ) num_reqs = swa_metadata.num_decodes + swa_metadata.num_prefills lookback_token_ids = input_ids.new_full( (num_reqs, self.engram_hash.lookback_depth), -1 ) engram_hashes = self.engram_hash( input_ids, positions, swa_metadata.query_start_loc, image_mask, lookback_token_ids, image_sentinel_mask(lookback_token_ids), swa_metadata.slot_mapping, swa_metadata.block_table, ) elif not self.engram_dp_shared_memory and get_engram_dp_size() > 1: # DP-sharded lookups are collective, so a replica skipping the # hash still has to reach them. engram_hashes, engram_mask = self.engram_hash.dummy_hashes(input_ids) if engram_hashes is not None: # Gather all Engram rows before entering the decoder layers. # One gather feeds every layer sharing the DP-split table. if self.engram_batch is not None: self.engram_batch.prepare_embeddings(engram_hashes) else: gathered_hashes = gather_engram_hashes( engram_hashes, dp_shared_memory=self.engram_dp_shared_memory ) for layer in islice(self.layers, self.start_layer, self.end_layer): engram = getattr(layer, "engram", None) if engram is not None: engram.prepare_embeddings( gathered_hashes[:, engram.layer_hash_index] ) full_num_tokens = positions.shape[0] if self.use_sequence_parallel: if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available(): forward_context = get_forward_context() forward_context.is_padding = sp_padding_mask( forward_context.is_padding, hidden_states ) hidden_states = sp_shard(hidden_states) input_ids = sp_shard(input_ids) if ( input_ids is not None and self.parallel_config.prefill_context_parallel_size > 1 and not self.parallel_config.enable_expert_parallel ): input_ids = get_pcp_group().all_gather(input_ids, dim=0) residual, post_mix, res_mix = None, None, None pre_mix: torch.Tensor | None = None if not get_pp_group().is_first_rank: assert intermediate_tensors is not None pre_mix = intermediate_tensors["pre_mix"] aux_hidden_by_layer: dict[int, torch.Tensor] = {} hidden_states, residual, post_mix, res_mix, pre_mix = self._run_layers( range(self.start_layer, self.decoder_replay_start), hidden_states, positions, input_ids, pre_mix, post_mix, res_mix, residual, aux_hidden_by_layer, engram_hashes, engram_mask, ) late_aux: list[torch.Tensor] = [] if self.decoder_replay_layers is not None: hidden_states, pre_mix, *late_aux = self.decoder_replay_layers( hidden_states, positions, input_ids, pre_mix, post_mix, res_mix, residual, ) else: hidden_states = self._collapse( hidden_states, residual, post_mix, res_mix, aux_hidden_by_layer, full_num_tokens, ) aux_hidden_states = [ aux_hidden_by_layer[layer_id] for layer_id in self.aux_hidden_state_layers if layer_id in aux_hidden_by_layer ] + late_aux if not get_pp_group().is_last_rank: return IntermediateTensors( {"hidden_states": hidden_states, "pre_mix": pre_mix} ) # MTP needs full HC states; otherwise collapse and normalize locally # before gathering to reduce communication. if self._mtp_hidden_buffer is not None: if self.use_sequence_parallel: hidden_states = sp_all_gather(hidden_states)[:full_num_tokens] pre_mix = sp_all_gather(pre_mix)[:full_num_tokens] num_tokens = hidden_states.shape[0] self._mtp_hidden_buffer[:num_tokens].copy_(hidden_states.flatten(1)) # Collapse the hc copies with the pre-mix from the last layer's FFN # mixes — the mix the reference applies via # ``last_layer.hc_pre(h, pre_mix)`` (v4.1 has no learned hc_head). assert pre_mix is not None hidden_states = hc_collapse_triton(hidden_states, pre_mix) hidden_states = self.norm(hidden_states) if self.use_sequence_parallel and self._mtp_hidden_buffer is None: # Without MTP, gather only the collapsed and normalized hidden states. hidden_states = sp_all_gather(hidden_states)[:full_num_tokens] if len(aux_hidden_states) > 0: return hidden_states, aux_hidden_states return hidden_states def _mega_gate_metadata( self, input_ids: torch.Tensor | None ) -> MegaGateRoutingMetadata | None: if not self.use_native_mega_moe: return None assert input_ids is not None return prepare_mega_gate_routing_metadata( input_ids, has_hash_routing=False, image_sentinel_base_id=IMAGE_SENTINEL_BASE_ID if getattr(self.config, "vision_n_layers", 0) > 0 else None, ) def _run_layers( self, layer_ids: range, hidden_states: torch.Tensor | MoEOutput, positions: torch.Tensor, input_ids: torch.Tensor | None, pre_mix: torch.Tensor | None, post_mix: torch.Tensor | None, res_mix: torch.Tensor | None, residual: torch.Tensor | None, aux_hidden_by_layer: dict[int, torch.Tensor], engram_hashes: torch.Tensor | None = None, engram_mask: torch.Tensor | None = None, ) -> tuple[ torch.Tensor | MoEOutput, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor ]: # Every layer's post runs inside the next layer's fused pre, so aux # hidden states are read back from there instead of recomputed. full_num_tokens = positions.shape[0] mega_gate_metadata = self._mega_gate_metadata(input_ids) for idx in layer_ids: hidden_states, residual, post_mix, res_mix, pre_mix, previous_aux = ( self.layers[idx]( hidden_states, positions, input_ids, pre_mix, post_mix, res_mix, residual, engram_hashes, engram_mask, capture_previous_aux=idx in self.aux_hidden_state_layers, mega_gate_metadata=mega_gate_metadata, ) ) if previous_aux is not None: # idx is the one-based id of the layer whose post this is. if self.use_sequence_parallel: previous_aux = sp_all_gather(previous_aux)[:full_num_tokens] aux_hidden_by_layer[idx] = previous_aux assert residual is not None and post_mix is not None assert res_mix is not None and pre_mix is not None return hidden_states, residual, post_mix, res_mix, pre_mix def _collapse( self, hidden_states: torch.Tensor | MoEOutput, residual: torch.Tensor, post_mix: torch.Tensor, res_mix: torch.Tensor, aux_hidden_by_layer: dict[int, torch.Tensor], full_num_tokens: int, ) -> torch.Tensor: # Without a successor boundary, the last layer finalized its own MoE. assert isinstance(hidden_states, torch.Tensor) # The last layer has no successor to fold its post into. if self.fuse_mhc_all_reduce: hidden_states = tensor_model_parallel_all_reduce(hidden_states) hidden_states = mhc_post_tilelang(hidden_states, residual, post_mix, res_mix) if self.end_layer in self.aux_hidden_state_layers: final_aux = hidden_states.mean(dim=1) if self.use_sequence_parallel: final_aux = sp_all_gather(final_aux)[:full_num_tokens] aux_hidden_by_layer[self.end_layer] = final_aux return hidden_states def _run_replay_layers( self, hidden_states: torch.Tensor | MoEOutput, positions: torch.Tensor, input_ids: torch.Tensor | None, pre_mix: torch.Tensor, post_mix: torch.Tensor, res_mix: torch.Tensor, residual: torch.Tensor, ) -> tuple[torch.Tensor, ...]: """Layers ``decoder_replay_start``.. on the given rows; returns their output, the last FFN's pre-mix and the aux hidden states they capture.""" aux_hidden_by_layer: dict[int, torch.Tensor] = {} hidden_states, residual, post_mix, res_mix, pre_mix = self._run_layers( range(self.decoder_replay_start, self.end_layer), hidden_states, positions, input_ids, pre_mix, post_mix, res_mix, residual, aux_hidden_by_layer, ) hidden_states = self._collapse( hidden_states, residual, post_mix, res_mix, aux_hidden_by_layer, positions.shape[0], ) return ( hidden_states, pre_mix, *( aux_hidden_by_layer[layer_id] for layer_id in self.aux_hidden_state_layers if layer_id in aux_hidden_by_layer ), ) def _decoder_replay_supported(self, vllm_config: VllmConfig, cut: int) -> bool: """Whether this rank may replay from layer ``cut`` on; warns when not.""" parallel_config = vllm_config.parallel_config spec_config = vllm_config.speculative_config draft_config = spec_config.draft_model_config if spec_config else None draft_hf_config = getattr(draft_config, "hf_config", None) draft_window = getattr(draft_hf_config, "sliding_window", None) draft_layer_types = getattr(draft_hf_config, "layer_types", None) or () window = self.config.sliding_window if self.start_layer > cut or self.end_layer < self.config.num_hidden_layers: reason = ( "the pipeline stage holding the last KV source layer must also " "hold every layer after it" ) elif ( self.use_sequence_parallel or parallel_config.prefill_context_parallel_size > 1 or parallel_config.use_ubatching ): reason = ( "the replay-layer batch shrinks per rank, which sequence and " "prefill-context parallelism and microbatching cannot follow" ) elif any(i >= cut for i in getattr(self.config, "engram_layer_ids", ())): reason = "an Engram layer sits at or after the last KV source layer" elif draft_config is not None and ( draft_window is None or draft_window > window or any(t != "sliding_attention" for t in draft_layer_types) ): reason = ( f"the drafter (sliding window {draft_window}) reads hidden states " f"outside the target's {window}-token window" ) else: return True logger.warning_once("Decoder SWA bounded replay is off: %s.", reason) return False def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: stacked_params_mapping = [ # (param_name, shard_name, shard_id) ("gate_up_proj", "w1", 0), ("gate_up_proj", "w3", 1), ("attn.fused_wqa_wkv", "attn.wq_a", 0), ("attn.fused_wqa_wkv", "attn.wkv", 1), ("compressor.fused_wkv_wgate", "compressor.wkv", 0), ("compressor.fused_wkv_wgate", "compressor.wgate", 1), ] params_dict = dict(self.named_parameters()) loaded_params: set[str] = set() # Pre-compute expert mapping ONCE. expert_mapping = self.get_expert_mapping() # Block-FP8 shared experts: pad the intermediate up to the TP-uniform # block count so the standard loaders below slice it evenly (trailing # ranks land on the zero pad). SP / unquantized ones need no padding. pad_shared_expert = ( getattr(self.quant_config, "weight_block_size", None) is not None and not self.use_sequence_parallel ) for name, loaded_weight in weights: if name.startswith(("vision.", "aligner.", "image_")): # Vision weights are loaded by the outer multimodal wrapper. logger.warning_once("Skipping non-text weight: %s", name) continue if ".shared_experts." in name: if is_pp_missing_parameter(name, self): continue experts = self.layers[extract_layer_index(name)].ffn.experts if getattr(experts, "has_fused_shared_experts", False): loaded_params.add( experts.load_shared_expert_weight(name, loaded_weight) ) continue if pad_shared_expert and ".shared_experts." in name: loaded_weight = self._pad_shared_expert_weight( self.quant_config, name, loaded_weight ) for param_name, weight_name, shard_id in stacked_params_mapping: # Skip non-stacked layers and experts (experts handled below). if ".experts." in name: continue if weight_name not in name: continue name = name.replace(weight_name, param_name) if is_pp_missing_parameter(name, self): break if name not in params_dict: head, _, leaf = name.rpartition(".") suffixed = f"{head}.base_layer.{leaf}" if suffixed in params_dict: name = suffixed param = params_dict[name] weight_loader = param.weight_loader weight_loader(param, loaded_weight, shard_id) loaded_params.add(name) break else: if ".experts." in name: # E8M0 scales are stored as float8_e8m0fnu in # checkpoints but the MoE param is uint8. copy_() # would do a numeric conversion (e.g. 2^-7 → 0), # destroying the raw exponent bytes. if ( "weight_scale" in name and loaded_weight.dtype == torch.float8_e8m0fnu ): loaded_weight = loaded_weight.view(torch.uint8) for mapping in expert_mapping: param_name, weight_name, expert_id, expert_shard_id = mapping if weight_name not in name: continue name_mapped = name.replace(weight_name, param_name) if is_pp_missing_parameter(name_mapped, self): continue param = params_dict[name_mapped] # We should ask the weight loader to return success or not # here since otherwise we may skip experts with other # available replicas. weight_loader = typing.cast( Callable[..., bool], param.weight_loader ) success = weight_loader( param, loaded_weight, name_mapped, shard_id=expert_shard_id, expert_id=expert_id, return_success=True, ) if success: name = name_mapped break loaded_params.add(name_mapped) continue else: if is_pp_missing_parameter(name, self): continue # Non-LoRA params on a LoRA-wrapped module live at # ``
.base_layer.