# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """DSpark draft model for DeepSeek-V4.1 (semi-autoregressive speculative decoding). See: qwen3_dspark.py for base architecture. This one is specialized to the DSV4.1 DSpark, which reuses the target model's architecture similarly to MTP. Its weights ship in the target checkpoint under the ``mtp.{0,1,2}.*`` prefix. To implement non-causal attention, we leverage the sparse attention implementation to include the future query tokens in the top-k indices for each query token. """ from collections.abc import Iterable import regex as re import torch import torch.nn as nn import vllm.envs as envs from vllm.config import VllmConfig, get_current_vllm_config from vllm.forward_context import get_forward_context, is_forward_context_available from vllm.logger import init_logger from vllm.model_executor.kernels.mhc.torch import mhc_post_torch from vllm.model_executor.layers.fused_moe import ( fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.qwen3_dspark import ( DSparkConfidenceHead, DSparkMarkovHead, ) from vllm.model_executor.models.utils import maybe_prefix from vllm.models.common.ops.sequence_parallel import ( sp_all_gather, sp_padding_mask, sp_shard, ) from .model import ( DeepseekV4DecoderLayer, DeepseekV4Model, _linear_scale_param_name, _use_sequence_parallel, ) logger = init_logger(__name__) # MoE expert scale suffix differs by expert dtype (mirrors deepseek_v4 loaders): # fp4 experts register ``.weight_scale``; block-fp8 experts ``.weight_scale_inv``. _EXPERT_SCALE_RE = re.compile(r"\.experts\.\d+\.w[123]\.scale$") class DSparkDeepseekV4Model(nn.Module): def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: super().__init__() assert vllm_config.speculative_config is not None config = vllm_config.speculative_config.draft_model_config.hf_config self.config = config self.hidden_size = config.hidden_size self.hc_mult = config.hc_mult self.hc_eps = config.hc_eps self.rms_norm_eps = config.rms_norm_eps self.num_hidden_layers = config.num_hidden_layers self.target_layer_ids = tuple(config.dspark_target_layer_ids) self.use_sequence_parallel = _use_sequence_parallel(vllm_config) self.num_dspark_layers = ( getattr(config, "n_mtp_layers", None) or getattr(config, "num_nextn_predict_layers", None) or 3 ) # Shared with the target (aliased by the speculator's loading utility). self.embed_tokens = VocabParallelEmbedding( config.vocab_size, config.hidden_size, prefix=maybe_prefix(prefix, "embed_tokens"), ) self.main_proj = ReplicatedLinear( config.hidden_size * len(self.target_layer_ids), config.hidden_size, bias=False, return_bias=False, quant_config=vllm_config.quant_config, prefix=maybe_prefix(prefix, "main_proj"), ) self.main_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.topk_indices_buffer = torch.empty( vllm_config.scheduler_config.max_num_batched_tokens, config.index_topk, dtype=torch.int32, ) current_vllm_config = get_current_vllm_config() self.layers = nn.ModuleList( [ DeepseekV4DecoderLayer( current_vllm_config, prefix=maybe_prefix(prefix, f"layers.{self.num_hidden_layers + i}"), topk_indices_buffer=self.topk_indices_buffer, ) for i in range(self.num_dspark_layers) ] ) # Heads: final norm, and the Markov + confidence heads. # Loaded from the "final" MTP layer weights (mtp.*) in the target # checkpoint. v4.1 has no learned hc_head: the hc copies are # collapsed with a pre-mix derived from the last layer's hc_ffn # projection at the end of forward(). self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) draft_vocab_size = ( getattr(config, "draft_vocab_size", None) or config.vocab_size ) self.markov_head = DSparkMarkovHead( config.vocab_size, draft_vocab_size, config.dspark_markov_rank, prefix=maybe_prefix(prefix, "markov_head"), ) self.confidence_head: DSparkConfidenceHead | None = None if getattr(config, "enable_confidence_head", True): self.confidence_head = DSparkConfidenceHead( config.hidden_size + config.dspark_markov_rank, prefix=maybe_prefix(prefix, "confidence_head"), ) def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.embed_tokens(input_ids) def combine_hidden_states(self, aux_hidden_states: torch.Tensor) -> torch.Tensor: """main_x = main_norm(main_proj(concat of target aux hidden states)). ``aux_hidden_states`` is [T, hidden_size * len(target_layer_ids)]. """ return self.main_norm(self.main_proj(aux_hidden_states)) @torch.inference_mode() def precompute_and_store_context_kv( self, main_x: torch.Tensor, context_positions: torch.Tensor, context_slot_mappings: list[torch.Tensor | None] | None = None, ) -> None: """Insert the sliding-window context KV for every draft layer. Mirrors the reference DSparkAttention: each layer derives its context KV from the SAME projected target hidden ``main_x``, via that layer's own ``wkv`` + ``kv_norm`` + RoPE + quant, then writes it at the layer's context slots. ``context_slot_mappings`` is a per-layer list (each entry is the context slot mapping for that layer's kv-cache group, since the hybrid manager may place draft layers in different groups). ``None`` (or a ``None`` entry) runs the projection to reserve workspace but writes nothing (profiling). """ for i, layer in enumerate(self.layers): slot_mapping = ( None if context_slot_mappings is None else context_slot_mappings[i] ) attn = layer.attn # Optimized DSV4 MLA path: wkv part of the fused wq_a|wkv projection # (q_lora part discarded), then RoPE/quant/insert via the fused op. qr_kv, _ = attn.fused_wqa_wkv(main_x) kv = qr_kv[..., attn.q_lora_rank :] kv = attn.kv_norm(kv) if slot_mapping is None: continue _insert_context_kv(attn, kv, context_positions, slot_mapping) def forward( self, input_ids: torch.Tensor, positions: torch.Tensor, inputs_embeds: torch.Tensor | None = None, ) -> torch.Tensor: if inputs_embeds is None: inputs_embeds = self.embed_input_ids(input_ids) full_num_tokens = positions.shape[0] if self.use_sequence_parallel: if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available(): forward_context = get_forward_context() forward_context.is_padding = sp_padding_mask( forward_context.is_padding, inputs_embeds ) inputs_embeds = sp_shard(inputs_embeds) input_ids = sp_shard(input_ids) # Expand to hc_mult copies for hyper-connections ([T, H] -> [T, hc, H]). hidden_states = inputs_embeds.unsqueeze(-2).repeat(1, self.hc_mult, 1) residual = post_mix = res_mix = pre_mix = None for layer in self.layers: hidden_states, residual, post_mix, res_mix, pre_mix = layer( hidden_states, positions, input_ids, pre_mix, post_mix, res_mix, residual, ) hidden_states = mhc_post_torch(hidden_states, residual, post_mix, res_mix) if self.use_sequence_parallel: hidden_states = sp_all_gather(hidden_states)[:full_num_tokens] pre_mix = sp_all_gather(pre_mix)[:full_num_tokens] # Collapse the hc copies with the pre-mix from the last layer's FFN # mixes — the mix the reference forward_head applies (hc_pre with the # last block's ffn pre-mix). Return the PRE-norm head hidden; # compute_logits applies self.norm. hidden_states = DeepseekV4DecoderLayer._hc_collapse(hidden_states, pre_mix) return hidden_states def _insert_context_kv( attn: nn.Module, kv: torch.Tensor, positions: torch.Tensor, slot_mapping: torch.Tensor, ) -> None: """RoPE + quant + paged-cache insert of (already kv_norm'd) context KV. Reuses the DSV4 fused insert ops (which also process a query; we pass a dummy query and discard it, since context tokens have no query). Mirrors ``DeepseekV4Attention._fused_qnorm_rope_kv_insert``. """ swa_cache = attn.swa_cache_layer.kv_cache block_size = attn.swa_cache_layer.block_size cos_sin_cache = attn.rotary_emb.cos_sin_cache cache_dtype = swa_cache.dtype n_ctx = kv.shape[0] dummy_q = torch.zeros( (n_ctx, attn.n_local_heads, attn.head_dim), dtype=kv.dtype, device=kv.device, ) if cache_dtype == torch.uint8: # fp8_ds_mla UE8M0 paged layout swa_2d = swa_cache.view(swa_cache.shape[0], -1) torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert( dummy_q, kv, swa_2d, slot_mapping, positions, cos_sin_cache, attn.padded_heads, attn.eps, block_size, True, # apply_q_norm (unused: the query is a discarded dummy) attn.kv_mxfp8, ) elif cache_dtype == torch.bfloat16: swa_3d = swa_cache.view(-1, block_size, attn.head_dim) torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert( dummy_q, kv, swa_3d, slot_mapping, positions, cos_sin_cache, attn.eps, block_size, ) else: # per-tensor fp8 (torch.float8_e4m3fn) # TODO(ben): double-check if this is being dispatched correctly for FI backend swa_3d = swa_cache.view(-1, block_size, attn.head_dim) dummy_q_fp8 = torch.zeros_like(dummy_q, dtype=torch.float8_e4m3fn) torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert( dummy_q, kv, dummy_q_fp8, swa_3d, slot_mapping, positions, cos_sin_cache, attn._flashinfer_fp8_kv_scale, attn._flashinfer_fp8_q_scale_inv, attn.eps, block_size, ) class DSparkDeepseekV4ForCausalLM(nn.Module): # Draft weights ship in the target checkpoint (mtp.*) without embed/head, so # load_dspark_model always aliases the target's. has_own_embed_tokens = False has_own_lm_head = False # Full-vocab draft: draft ids are target ids, no remapping needed. draft_id_to_target_id = None def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: super().__init__() assert vllm_config.speculative_config is not None self.draft_model_config = vllm_config.speculative_config.draft_model_config self.config = self.draft_model_config.hf_config self.quant_config = vllm_config.quant_config self.linear_scale_name = _linear_scale_param_name( vllm_config, getattr(self.config, "expert_dtype", "fp4") ) self.pad_shared_expert = getattr( self.quant_config, "weight_block_size", None ) is not None and not _use_sequence_parallel(vllm_config) self.model = DSparkDeepseekV4Model( vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") ) # Shared with the target (aliased by the speculator's load utility). self.lm_head = ParallelLMHead( self.config.vocab_size, self.config.hidden_size, prefix=maybe_prefix(prefix, "lm_head"), ) self.logits_processor = LogitsProcessor(self.config.vocab_size) # --- Hooks used by the speculator ------------------------------------- def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.model.embed_input_ids(input_ids) def combine_hidden_states(self, aux_hidden_states: torch.Tensor) -> torch.Tensor: return self.model.combine_hidden_states(aux_hidden_states) def get_draft_kv_cache_layer_names(self) -> list[str]: # DSV4 MLA path: each draft layer's sliding-window cache is a separate # layer, named by its prefix. return [layer.attn.swa_cache_layer.prefix for layer in self.model.layers] def precompute_and_store_context_kv( self, context_states: torch.Tensor, context_positions: torch.Tensor, context_slot_mappings: list[torch.Tensor | None] | None = None, ) -> None: self.model.precompute_and_store_context_kv( context_states, context_positions, context_slot_mappings ) def forward( self, input_ids: torch.Tensor, positions: torch.Tensor, inputs_embeds: torch.Tensor | None = None, ) -> torch.Tensor: # Returns the pre-norm collapsed head hidden ([T, hidden_size]). return self.model(input_ids, positions, inputs_embeds) def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor: """Base logits U_k = lm_head(norm(head_hidden)).""" return self.logits_processor(self.lm_head, self.model.norm(hidden_states)) def compute_draft_logits(self, hidden_states: torch.Tensor) -> torch.Tensor: # Full-vocab draft: base logits, no d2t scatter. return self.compute_logits(hidden_states) def map_draft_to_target(self, draft_ids: torch.Tensor) -> torch.Tensor: return draft_ids # full-vocab: draft ids are target ids def markov_embed(self, token_ids: torch.Tensor) -> torch.Tensor: return self.model.markov_head.embed(token_ids) def markov_bias(self, markov_embed: torch.Tensor) -> torch.Tensor: return self.model.markov_head.bias(markov_embed, self.logits_processor) def compute_confidence( self, head_hidden: torch.Tensor, markov_embed: torch.Tensor ) -> torch.Tensor: """Per-position acceptance probability for each drafted token.""" assert self.model.confidence_head is not None return torch.sigmoid(self.model.confidence_head(head_hidden, markov_embed)) # --- Weight loading ---------------------------------------------------- def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: """Load the ``mtp.{0,1,2}.*`` draft weights from the target checkpoint. Non-mtp weights (embed/head/main layers) belong to the target model and are skipped here. ``embed_tokens``/``lm_head`` are aliased from the target. """ # Draft MoE layers use the dspark_* expert counts, not the # backbone's (see DeepseekV4MoE and the reference # ModelArgs.get_moe_config). n_draft_experts = ( getattr(self.config, "dspark_n_routed_experts", 0) or self.config.n_routed_experts ) expert_mapping = fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="w1", ckpt_down_proj_name="w2", ckpt_up_proj_name="w3", num_experts=n_draft_experts, ) expert_scale_suffix = ( ".weight_scale" if getattr(self.config, "expert_dtype", "fp4") == "fp4" else ".weight_scale_inv" ) # (param_name, ckpt_shard_name, shard_id) for non-expert stacked params. stacked_params_mapping = [ ("gate_up_proj", "w1", 0), ("gate_up_proj", "w3", 1), ("attn.fused_wqa_wkv", "attn.wq_a", 0), ("attn.fused_wqa_wkv", "attn.wkv", 1), ] params_dict = dict(self.named_parameters()) loaded_params: set[str] = set() loaded_confidence_head = False for name, loaded_weight in weights: mapped = self._remap_dspark_name(name) if mapped is None: continue name = mapped if "confidence_head." in name: loaded_confidence_head = True # ``.scale`` -> per-method scale suffix. if name.endswith(".scale"): suffix = ( expert_scale_suffix if _EXPERT_SCALE_RE.search(name) else f".{self.linear_scale_name}" ) name = name.removesuffix(".scale") + suffix if ".shared_experts.w2" in name: name = name.replace(".shared_experts.w2", ".shared_experts.down_proj") if self.pad_shared_expert and ".shared_experts." in name: loaded_weight = DeepseekV4Model._pad_shared_expert_weight( self.quant_config, name, loaded_weight ) # E8M0 expert scales: keep raw exponent bytes. if ".experts." in name: if ( "weight_scale" in name and loaded_weight.dtype == torch.float8_e8m0fnu ): loaded_weight = loaded_weight.view(torch.uint8) for param_name, weight_name, expert_id, shard_id in expert_mapping: if weight_name not in name: continue name_mapped = name.replace(weight_name, param_name) param = params_dict[name_mapped] success = param.weight_loader( param, loaded_weight, name_mapped, shard_id=shard_id, expert_id=expert_id, return_success=True, ) if success: loaded_params.add(name_mapped) break continue # Stacked rules only apply to decoder-layer weights. Head-stack params # (main_proj/norm/markov_head/confidence_head) load directly — # otherwise e.g. "markov_w1" would collide with the "w1" shard rule. is_layer_param = name.startswith("model.layers.") for param_name, weight_name, stacked_shard_id in stacked_params_mapping: if not is_layer_param or weight_name not in name: continue name = name.replace(weight_name, param_name) param = params_dict[name] param.weight_loader(param, loaded_weight, stacked_shard_id) loaded_params.add(name) break else: if name.endswith(".ffn.gate.bias"): name = name.replace( ".ffn.gate.bias", ".ffn.gate.e_score_correction_bias" ) param = params_dict[name] weight_loader = getattr(param, "weight_loader", default_weight_loader) weight_loader(param, loaded_weight) loaded_params.add(name) if self.model.confidence_head is not None and not loaded_confidence_head: self.model.confidence_head = None self.process_weights_after_loading() logger.info_once("DSpark draft model loaded: %d params", len(loaded_params)) return loaded_params def process_weights_after_loading(self) -> None: # ROCm linears and fused-MoE runners finalize their own quantized # parameters. NVIDIA-only MegaMoE/WO-A requantization is not needed. return def _remap_dspark_name(self, name: str) -> str | None: """Map a checkpoint ``mtp.{i}.*`` name to this model's parameter path. Returns None for non-mtp weights (owned by the target model). """ m = re.match(r"mtp\.(\d+)\.(.*)", name) if m is None: return None stage = int(m.group(1)) rest = m.group(2) if rest.startswith("confidence_head.") and self.model.confidence_head is None: return None # The checkpoint calls the Markov head's factors ``embed``/``head``; # DSparkMarkovHead registers them as ``markov_w1``/``markov_w2``. if rest.startswith("markov_head.embed."): return "model.markov_head.markov_w1." + rest.removeprefix( "markov_head.embed." ) if rest.startswith("markov_head.head."): return "model.markov_head.markov_w2." + rest.removeprefix( "markov_head.head." ) # Head-stack params live at model level (mtp.last), context combiner at # model level (mtp.0); everything else is a per-layer decoder block. head_prefixes = ( "norm.", "markov_head.", "confidence_head.", ) if rest.startswith(("main_proj.", "main_norm.")) or rest.startswith( head_prefixes ): return f"model.{rest}" return f"model.layers.{stage}.{rest}"