From 9969466a597810db6e06b4942dd6cc2086885ee2 Mon Sep 17 00:00:00 2001 From: Benjamin Chislett Date: Tue, 30 Jun 2026 23:34:47 -0400 Subject: [PATCH] [Spec Decode] Support SWA + DFlash for MiMo (#46104) --- tests/models/registry.py | 2 +- vllm/model_executor/models/mimo_v2.py | 19 +- vllm/model_executor/models/qwen3_dflash.py | 197 ++++++++++++++++++++- vllm/v1/attention/backends/flash_attn.py | 50 ++++-- 4 files changed, 243 insertions(+), 25 deletions(-) diff --git a/tests/models/registry.py b/tests/models/registry.py index 727a12d045..14bc15ee14 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -1408,7 +1408,7 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = { # [DFlash] "DFlashDraftModel": _HfExamplesInfo( "Qwen/Qwen3.5-4B", - speculative_model="z-lab/Qwen3.5-4B-DFlash", + speculative_model="z-lab/Qwen3-4B-DFlash-b16", use_original_num_layers=True, # Need all layers since DFlash has >1 layer, max_model_len=8192, # Reduce max len to ensure test runs in low-VRAM CI env max_num_seqs=32, diff --git a/vllm/model_executor/models/mimo_v2.py b/vllm/model_executor/models/mimo_v2.py index 84459df4d2..4c2ebd958b 100644 --- a/vllm/model_executor/models/mimo_v2.py +++ b/vllm/model_executor/models/mimo_v2.py @@ -53,7 +53,12 @@ from vllm.sequence import IntermediateTensors from vllm.v1.attention.backend import AttentionType from vllm.v1.attention.backends.registry import AttentionBackendEnum -from .interfaces import MixtureOfExperts, SupportsPP +from .interfaces import ( + EagleModelMixin, + MixtureOfExperts, + SupportsEagle3, + SupportsPP, +) from .utils import ( AutoWeightsLoader, PPMissingLayer, @@ -539,7 +544,7 @@ def _shard_fp8_qkv_proj( @support_torch_compile -class MiMoV2Model(nn.Module): +class MiMoV2Model(nn.Module, EagleModelMixin): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() @@ -602,10 +607,16 @@ class MiMoV2Model(nn.Module): hidden_states = intermediate_tensors["hidden_states"] residual = intermediate_tensors["residual"] + aux_hidden_states = self._maybe_add_hidden_state( + [], self.start_layer, hidden_states, residual + ) for idx, layer in enumerate( islice(self.layers, self.start_layer, self.end_layer) ): hidden_states, residual = layer(positions, hidden_states, residual) + self._maybe_add_hidden_state( + aux_hidden_states, idx + 1, hidden_states, residual + ) if not get_pp_group().is_last_rank: return IntermediateTensors( @@ -614,6 +625,8 @@ class MiMoV2Model(nn.Module): hidden_states, _ = self.norm(hidden_states, residual) + if len(aux_hidden_states) > 0: + return hidden_states, aux_hidden_states return hidden_states def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: @@ -822,7 +835,7 @@ class MiMoV2Model(nn.Module): return True -class MiMoV2FlashForCausalLM(nn.Module, SupportsPP, MixtureOfExperts): +class MiMoV2FlashForCausalLM(nn.Module, SupportsPP, MixtureOfExperts, SupportsEagle3): packed_modules_mapping = { "qkv_proj": ["q_proj", "k_proj", "v_proj"], "gate_up_proj": ["gate_proj", "up_proj"], diff --git a/vllm/model_executor/models/qwen3_dflash.py b/vllm/model_executor/models/qwen3_dflash.py index 8746a15f11..99c16f49df 100644 --- a/vllm/model_executor/models/qwen3_dflash.py +++ b/vllm/model_executor/models/qwen3_dflash.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import io from collections.abc import Iterable import torch @@ -11,7 +12,10 @@ from transformers import Qwen3Config from vllm import _custom_ops as ops from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config -from vllm.distributed import get_tensor_model_parallel_world_size +from vllm.distributed import ( + get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, +) from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.layernorm import RMSNorm @@ -33,6 +37,7 @@ from vllm.model_executor.model_loader.weight_utils import ( ) from vllm.multimodal.inputs import NestedTensors from vllm.transformers_utils.config import set_default_rope_theta +from vllm.transformers_utils.repo_utils import get_hf_file_bytes from vllm.v1.attention.backend import AttentionType from .qwen2 import Qwen2MLP as Qwen3MLP @@ -47,6 +52,76 @@ from .utils import ( logger = init_logger(__name__) +def _resolve_layer_attention( + config: Qwen3Config, layer_idx: int +) -> tuple[int | None, bool]: + """Resolve ``(sliding_window, causal)`` for one DFlash draft layer. + + +----------------------+-------------------------+--------------------------------+ + | Config | ``layer_type`` | *``causal`` | + +======================+=========================+================================+ + | ``layer_types`` | SWA if ``use_swa`` | True if ``layer_types[i]=SWA`` | + | | else ``layer_types[i]`` | else False | + +----------------------+-------------------------+--------------------------------+ + | ``layer_types=None`` | SWA | False | + | + ``use_swa=True`` | | | + +----------------------+-------------------------+--------------------------------+ + | ``layer_types=None`` | Full | False | + | + ``use_swa=False`` | | | + +----------------------+-------------------------+--------------------------------+ + * If ``dflash_config.causal`` is set, its value overrides ``causal`` for all layers. + + This is to support a varied ecosystem of checkpoints, including: + - XiaomiMiMo/MiMo-V2.5-Pro-FP4-DFlash (sets "use_swa", assumes non-causal) + - z-lab/gemma-4-31B-it-DFlash (has mixed layer types, assumes causal only for SWA) + - z-lab/Qwen3.5-9B-DFlash ("standard" DFlash, all full attn, assumes non-causal) + """ + dflash_config = getattr(config, "dflash_config", None) or {} + layer_types = getattr(config, "layer_types", None) + use_swa = dflash_config.get("use_swa", False) + config_causal = dflash_config.get("causal", None) + + SLIDING_ATTENTION = "sliding_attention" + any_sliding = False + if layer_types is not None: + num_sliding = sum(lt == SLIDING_ATTENTION for lt in layer_types) + any_sliding = num_sliding > 0 + all_sliding = num_sliding == len(layer_types) + if any_sliding and not all_sliding: + # Mixed sliding/full attention needs per-layer causal metadata and + # multiple KV-cache groups, which DFlash does not yet support. + raise NotImplementedError( + "DFlash does not yet support mixed sliding/full attention via " + "layer_types; see " + "https://github.com/vllm-project/vllm/issues/40898." + ) + + default_causal = False + if layer_types is None or (use_swa and not any_sliding): + # An absent ``layer_types`` (or the all-"full_attention" one that may + # be synthesized when the checkpoint omits it) must not override + # ``dflash_config.use_swa``, which forces SWA on every layer. + is_sliding = use_swa + else: + is_sliding = layer_types[layer_idx] == SLIDING_ATTENTION + # Full-attention layers default non-causal; SWA layers default causal. + default_causal = is_sliding + + sliding_window = None + if is_sliding: + sliding_window = dflash_config.get( + "swa_window_size", getattr(config, "sliding_window", None) + ) + if sliding_window is None: + raise ValueError( + "DFlash sliding attention requires a window size configured in " + "dflash_config.swa_window_size or the top-level sliding_window." + ) + + causal = config_causal if config_causal is not None else default_causal + return sliding_window, causal + + class DFlashQwen3Attention(nn.Module): """Attention for DFlash speculative decoding. @@ -64,6 +139,9 @@ class DFlashQwen3Attention(nn.Module): head_dim: int | None = None, rms_norm_eps: float = 1e-06, attention_bias: bool = False, + add_swa_attention_sink_bias: bool = False, + sliding_window: int | None = None, + causal: bool = False, cache_config: CacheConfig | None = None, quant_config: QuantizationConfig | None = None, prefix: str = "", @@ -109,6 +187,14 @@ class DFlashQwen3Attention(nn.Module): max_position=max_position, rope_parameters=rope_parameters, ) + + self.attention_sink_bias = ( + torch.nn.Parameter(torch.empty(self.num_heads), requires_grad=False) + if add_swa_attention_sink_bias + else None + ) + + self.sliding_window = sliding_window self.attn = Attention( self.num_heads, self.head_dim, @@ -116,9 +202,13 @@ class DFlashQwen3Attention(nn.Module): num_kv_heads=self.num_kv_heads, cache_config=cache_config, quant_config=quant_config, + per_layer_sliding_window=sliding_window, prefix=f"{prefix}.attn", attn_type=attn_type, + sinks=self.attention_sink_bias, ) + # NOTE: `causal` is currently unused here, but will be needed in the future + # to support models with different causality per-layer. self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) @@ -156,6 +246,7 @@ class DFlashQwen3DecoderLayer(nn.Module): vllm_config: VllmConfig, *, config: Qwen3Config, + layer_idx: int, cache_config: CacheConfig | None = None, quant_config: QuantizationConfig | None = None, prefix: str = "", @@ -165,6 +256,18 @@ class DFlashQwen3DecoderLayer(nn.Module): set_default_rope_theta(config, default_theta=1000000) attn_type = AttentionType.DECODER + # DFlash drafts store the sink-bias flag inside dflash_config; fall back + # to the top-level attribute used by other (e.g. MiMo) configs. + dflash_config = getattr(config, "dflash_config", None) or {} + add_swa_attention_sink_bias = dflash_config.get( + "attention_sink_bias", + getattr(config, "add_swa_attention_sink_bias", False), + ) + + # Resolve this layer's attention mode (full vs sliding window, causal vs + # non-causal) from the draft config. + sliding_window, causal = _resolve_layer_attention(config, layer_idx) + self.self_attn = DFlashQwen3Attention( hidden_size=self.hidden_size, num_heads=config.num_attention_heads, @@ -172,6 +275,9 @@ class DFlashQwen3DecoderLayer(nn.Module): num_kv_heads=config.num_key_value_heads, rms_norm_eps=config.rms_norm_eps, attention_bias=getattr(config, "attention_bias", False), + add_swa_attention_sink_bias=add_swa_attention_sink_bias, + sliding_window=sliding_window, + causal=causal, head_dim=getattr(config, "head_dim", None), cache_config=cache_config, quant_config=quant_config, @@ -243,11 +349,24 @@ class DFlashQwen3Model(nn.Module): prefix=maybe_prefix(prefix, "embed_tokens"), ) + # Masked query slots are fed to the draft as `mask_token_id`. Most DFlash + # checkpoints will have the mask embedding in the vocabulary embedding table + # at that slot id. Some checkpoints (XiaomiMiMo/MiMo-V2.5-Pro-FP4-DFlash) ship + # with a separate mask embedding tensor to use instead. When present, we load it + # and substitute it for embed_tokens[mask_token_id] when computing embeddings. + self.mask_token_id = drafter_config.get("mask_token_id") + self.mask_embedding = nn.Parameter( + torch.zeros(self.config.hidden_size, dtype=vllm_config.model_config.dtype), + requires_grad=False, + ) + self.has_separate_mask_embedding = False + self.layers = nn.ModuleList( [ DFlashQwen3DecoderLayer( current_vllm_config, config=self.config, + layer_idx=layer_idx, cache_config=current_vllm_config.cache_config, quant_config=self.quant_config, prefix=maybe_prefix(prefix, f"layers.{layer_idx + start_layer_id}"), @@ -284,7 +403,12 @@ class DFlashQwen3Model(nn.Module): ) def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: - return self.embed_tokens(input_ids) + embeds = self.embed_tokens(input_ids) + if self.has_separate_mask_embedding and self.mask_token_id is not None: + # Replace masked slots with the dedicated mask embedding. + is_mask = (input_ids == self.mask_token_id).unsqueeze(-1) + embeds = torch.where(is_mask, self.mask_embedding.to(embeds.dtype), embeds) + return embeds def _build_fused_kv_buffers(self) -> None: """Build fused weight buffers for precompute_and_store_context_kv. @@ -469,6 +593,8 @@ class DFlashQwen3Model(nn.Module): ] params_dict = dict(self.named_parameters()) loaded_params: set[str] = set() + tp_rank = get_tensor_model_parallel_rank() + tp_size = get_tensor_model_parallel_world_size() for name, loaded_weight in weights: if "midlayer." in name: name = name.replace("midlayer.", "layers.0.") @@ -476,6 +602,18 @@ class DFlashQwen3Model(nn.Module): name = maybe_remap_kv_scale_name(name, params_dict) if name is None: continue + if "attention_sink_bias" in name: + if name not in params_dict: + continue + # Sink bias is per-head; shard it across TP ranks like the + # attention heads themselves. + param = params_dict[name] + heads_per_rank = loaded_weight.shape[0] // tp_size + head_start = tp_rank * heads_per_rank + narrow_weight = loaded_weight.narrow(0, head_start, heads_per_rank) + param.data.copy_(narrow_weight) + loaded_params.add(name) + continue for param_name, weight_name, shard_id in stacked_params_mapping: if weight_name not in name: continue @@ -495,7 +633,8 @@ class DFlashQwen3Model(nn.Module): class DFlashQwen3ForCausalLM(Qwen3ForCausalLM): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): nn.Module.__init__(self) - self.config = vllm_config.speculative_config.draft_model_config.hf_config + self.draft_model_config = vllm_config.speculative_config.draft_model_config + self.config = self.draft_model_config.hf_config if getattr(self.config, "draft_vocab_size", None) is None: self.config.draft_vocab_size = getattr(self.config, "vocab_size", None) target_layer_num = vllm_config.model_config.get_num_layers( @@ -589,7 +728,9 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM): includes_embed_tokens = False for name, loaded_weight in weights: assert "mask_hidden" not in name, ( - "DFlash should use mask_token_id to embed the padding hidden state" + "DFlash embeds masked slots via mask_token_id (optionally " + "overridden by a mask_embedding.pt file); it should not ship a " + "mask_hidden weight." ) if "t2d" in name: continue @@ -603,6 +744,13 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM): model_weights[name] = loaded_weight process_eagle_weight(self, name) + # Route the separately-trained mask embedding (if shipped) through the + # standard weight loader alongside the rest of the draft weights. + mask_embedding = self._read_mask_embedding() + if mask_embedding is not None: + model_weights["model.mask_embedding"] = mask_embedding + self.model.has_separate_mask_embedding = True + skip_substrs = [] if not includes_draft_id_mapping: skip_substrs.append("draft_id_to_target_id") @@ -610,6 +758,8 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM): skip_substrs.append("embed_tokens") if not self.model.use_aux_hidden_state: skip_substrs.append("fc.") + if not self.model.has_separate_mask_embedding: + skip_substrs.append("mask_embedding") loader = AutoWeightsLoader( self, skip_prefixes=None, @@ -617,3 +767,42 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM): ) loader.load_weights(model_weights.items()) self.model._build_fused_kv_buffers() + + def _read_mask_embedding(self) -> torch.Tensor | None: + """Checks for an override mask embedding in `mask_embedding.pt` and returns it. + + Some checkpoints ship a separately-trained mask embedding for the mask token, + which we use to overwrite the embedding for `mask_token_id`. This helper + checks for the file, loads the pytorch tensor, and returns the embedding to use. + + Returns None if the override file is not present. + """ + mask_token_id = self.model.mask_token_id + if mask_token_id is None: + return None + + MASK_EMBEDDING_FILENAME = "mask_embedding.pt" + data = get_hf_file_bytes( + MASK_EMBEDDING_FILENAME, + self.draft_model_config.model, + self.draft_model_config.revision, + ) + if data is None: + return None + + state = torch.load(io.BytesIO(data), weights_only=True) + if isinstance(state, dict): + if state.get("mask_token_id", mask_token_id) != mask_token_id: + raise ValueError( + f"{MASK_EMBEDDING_FILENAME} mask_token_id does not match " + f"dflash_config.mask_token_id ({mask_token_id}). " + f"Got {state.get('mask_token_id')}." + ) + state = state["embedding"] + + logger.info( + "Loaded DFlash mask embedding for mask_token_id %s from %s", + mask_token_id, + MASK_EMBEDDING_FILENAME, + ) + return state.reshape(-1) diff --git a/vllm/v1/attention/backends/flash_attn.py b/vllm/v1/attention/backends/flash_attn.py index 2eed819056..7823fd48b5 100755 --- a/vllm/v1/attention/backends/flash_attn.py +++ b/vllm/v1/attention/backends/flash_attn.py @@ -252,6 +252,8 @@ class FlashAttentionMetadata: causal: bool | torch.Tensor = True + sliding_window: tuple[int, int] | None = None + # PrefixLM bidirectional ranges for multimodal tokens. # Shape: (num_seqs, max_ranges, 2) int32, [start, end] per range. mm_prefix_range_tensor: torch.Tensor | None = None @@ -284,6 +286,20 @@ def _get_sliding_window_configs( return sliding_window_configs +def _maybe_symmetrize_window( + window: tuple[int, int] | None, + causal: bool | torch.Tensor, +) -> tuple[int, int] | None: + """Make a causal sliding window ``(w, 0)`` symmetric ``(w, w)`` when attention + is non-causal, so bidirectional queries attend in both directions. Leaves + full-attention ``(-1, -1)`` and already-symmetric windows untouched. + """ + non_causal = isinstance(causal, torch.Tensor) or causal is False + if window is not None and window[0] >= 0 and window[1] == 0 and non_causal: + return (window[0], window[0]) + return window + + class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetadata]): # FA3: # Supports full cudagraphs for all cases. @@ -487,7 +503,9 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad cu_seqlens_q=cu_query_lens, page_size=self.block_size, causal=causal, - window_size=self.aot_sliding_window, + window_size=_maybe_symmetrize_window( + self.aot_sliding_window, causal + ), num_splits=max_num_splits, ) return None @@ -579,6 +597,13 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad if isinstance(causal, torch.Tensor) and causal.dtype != torch.int32: causal = causal.to(torch.int32) + # Symmetrize the spec's sliding_window for non-causal attention + group_sliding_window = getattr(self.kv_cache_spec, "sliding_window", None) + base_window = ( + (-1, -1) if group_sliding_window is None else (group_sliding_window - 1, 0) + ) + effective_sliding_window = _maybe_symmetrize_window(base_window, causal) + attn_metadata = FlashAttentionMetadata( num_actual_tokens=num_actual_tokens, max_query_len=max_query_len, @@ -598,6 +623,7 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad prefix_scheduler_metadata=prefix_scheduler_metadata, max_num_splits=max_num_splits, causal=causal, + sliding_window=effective_sliding_window, ) # Compute mm_prefix range tensor if the batch contains @@ -844,28 +870,18 @@ class FlashAttentionImpl(AttentionImpl): ) return output else: + window = ( + attn_metadata.sliding_window + if attn_metadata.sliding_window is not None + else self.sliding_window + ) sliding_window_size: list[int] | None = ( - list(self.sliding_window) - if self.sliding_window is not None - else None + list(window) if window is not None else None ) causal = attn_metadata.causal is_dynamic_causal = isinstance(causal, torch.Tensor) - # For non-causal (bidirectional) attention, make the - # sliding window symmetric so queries attend in both - # directions. - if ( - sliding_window_size is not None - and sliding_window_size[1] == 0 - and (is_dynamic_causal or causal is False) - ): - sliding_window_size = [ - sliding_window_size[0], - sliding_window_size[0], - ] - mm_prefix_ranges = attn_metadata.mm_prefix_range_tensor mm_mask_mod = None mm_aux = None