[Spec Decode] Support SWA + DFlash for MiMo (#46104)

This commit is contained in:
Benjamin Chislett
2026-06-30 20:34:47 -07:00
committed by GitHub
parent 3406e8f83d
commit 9969466a59
4 changed files with 243 additions and 25 deletions
+1 -1
View File
@@ -1408,7 +1408,7 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
# [DFlash]
"DFlashDraftModel": _HfExamplesInfo(
"Qwen/Qwen3.5-4B",
speculative_model="z-lab/Qwen3.5-4B-DFlash",
speculative_model="z-lab/Qwen3-4B-DFlash-b16",
use_original_num_layers=True, # Need all layers since DFlash has >1 layer,
max_model_len=8192, # Reduce max len to ensure test runs in low-VRAM CI env
max_num_seqs=32,
+16 -3
View File
@@ -53,7 +53,12 @@ from vllm.sequence import IntermediateTensors
from vllm.v1.attention.backend import AttentionType
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from .interfaces import MixtureOfExperts, SupportsPP
from .interfaces import (
EagleModelMixin,
MixtureOfExperts,
SupportsEagle3,
SupportsPP,
)
from .utils import (
AutoWeightsLoader,
PPMissingLayer,
@@ -539,7 +544,7 @@ def _shard_fp8_qkv_proj(
@support_torch_compile
class MiMoV2Model(nn.Module):
class MiMoV2Model(nn.Module, EagleModelMixin):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -602,10 +607,16 @@ class MiMoV2Model(nn.Module):
hidden_states = intermediate_tensors["hidden_states"]
residual = intermediate_tensors["residual"]
aux_hidden_states = self._maybe_add_hidden_state(
[], self.start_layer, hidden_states, residual
)
for idx, layer in enumerate(
islice(self.layers, self.start_layer, self.end_layer)
):
hidden_states, residual = layer(positions, hidden_states, residual)
self._maybe_add_hidden_state(
aux_hidden_states, idx + 1, hidden_states, residual
)
if not get_pp_group().is_last_rank:
return IntermediateTensors(
@@ -614,6 +625,8 @@ class MiMoV2Model(nn.Module):
hidden_states, _ = self.norm(hidden_states, residual)
if len(aux_hidden_states) > 0:
return hidden_states, aux_hidden_states
return hidden_states
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
@@ -822,7 +835,7 @@ class MiMoV2Model(nn.Module):
return True
class MiMoV2FlashForCausalLM(nn.Module, SupportsPP, MixtureOfExperts):
class MiMoV2FlashForCausalLM(nn.Module, SupportsPP, MixtureOfExperts, SupportsEagle3):
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
"gate_up_proj": ["gate_proj", "up_proj"],
+193 -4
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import io
from collections.abc import Iterable
import torch
@@ -11,7 +12,10 @@ from transformers import Qwen3Config
from vllm import _custom_ops as ops
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -33,6 +37,7 @@ from vllm.model_executor.model_loader.weight_utils import (
)
from vllm.multimodal.inputs import NestedTensors
from vllm.transformers_utils.config import set_default_rope_theta
from vllm.transformers_utils.repo_utils import get_hf_file_bytes
from vllm.v1.attention.backend import AttentionType
from .qwen2 import Qwen2MLP as Qwen3MLP
@@ -47,6 +52,76 @@ from .utils import (
logger = init_logger(__name__)
def _resolve_layer_attention(
config: Qwen3Config, layer_idx: int
) -> tuple[int | None, bool]:
"""Resolve ``(sliding_window, causal)`` for one DFlash draft layer.
+----------------------+-------------------------+--------------------------------+
| Config | ``layer_type`` | *``causal`` |
+======================+=========================+================================+
| ``layer_types`` | SWA if ``use_swa`` | True if ``layer_types[i]=SWA`` |
| | else ``layer_types[i]`` | else False |
+----------------------+-------------------------+--------------------------------+
| ``layer_types=None`` | SWA | False |
| + ``use_swa=True`` | | |
+----------------------+-------------------------+--------------------------------+
| ``layer_types=None`` | Full | False |
| + ``use_swa=False`` | | |
+----------------------+-------------------------+--------------------------------+
* If ``dflash_config.causal`` is set, its value overrides ``causal`` for all layers.
This is to support a varied ecosystem of checkpoints, including:
- XiaomiMiMo/MiMo-V2.5-Pro-FP4-DFlash (sets "use_swa", assumes non-causal)
- z-lab/gemma-4-31B-it-DFlash (has mixed layer types, assumes causal only for SWA)
- z-lab/Qwen3.5-9B-DFlash ("standard" DFlash, all full attn, assumes non-causal)
"""
dflash_config = getattr(config, "dflash_config", None) or {}
layer_types = getattr(config, "layer_types", None)
use_swa = dflash_config.get("use_swa", False)
config_causal = dflash_config.get("causal", None)
SLIDING_ATTENTION = "sliding_attention"
any_sliding = False
if layer_types is not None:
num_sliding = sum(lt == SLIDING_ATTENTION for lt in layer_types)
any_sliding = num_sliding > 0
all_sliding = num_sliding == len(layer_types)
if any_sliding and not all_sliding:
# Mixed sliding/full attention needs per-layer causal metadata and
# multiple KV-cache groups, which DFlash does not yet support.
raise NotImplementedError(
"DFlash does not yet support mixed sliding/full attention via "
"layer_types; see "
"https://github.com/vllm-project/vllm/issues/40898."
)
default_causal = False
if layer_types is None or (use_swa and not any_sliding):
# An absent ``layer_types`` (or the all-"full_attention" one that may
# be synthesized when the checkpoint omits it) must not override
# ``dflash_config.use_swa``, which forces SWA on every layer.
is_sliding = use_swa
else:
is_sliding = layer_types[layer_idx] == SLIDING_ATTENTION
# Full-attention layers default non-causal; SWA layers default causal.
default_causal = is_sliding
sliding_window = None
if is_sliding:
sliding_window = dflash_config.get(
"swa_window_size", getattr(config, "sliding_window", None)
)
if sliding_window is None:
raise ValueError(
"DFlash sliding attention requires a window size configured in "
"dflash_config.swa_window_size or the top-level sliding_window."
)
causal = config_causal if config_causal is not None else default_causal
return sliding_window, causal
class DFlashQwen3Attention(nn.Module):
"""Attention for DFlash speculative decoding.
@@ -64,6 +139,9 @@ class DFlashQwen3Attention(nn.Module):
head_dim: int | None = None,
rms_norm_eps: float = 1e-06,
attention_bias: bool = False,
add_swa_attention_sink_bias: bool = False,
sliding_window: int | None = None,
causal: bool = False,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
@@ -109,6 +187,14 @@ class DFlashQwen3Attention(nn.Module):
max_position=max_position,
rope_parameters=rope_parameters,
)
self.attention_sink_bias = (
torch.nn.Parameter(torch.empty(self.num_heads), requires_grad=False)
if add_swa_attention_sink_bias
else None
)
self.sliding_window = sliding_window
self.attn = Attention(
self.num_heads,
self.head_dim,
@@ -116,9 +202,13 @@ class DFlashQwen3Attention(nn.Module):
num_kv_heads=self.num_kv_heads,
cache_config=cache_config,
quant_config=quant_config,
per_layer_sliding_window=sliding_window,
prefix=f"{prefix}.attn",
attn_type=attn_type,
sinks=self.attention_sink_bias,
)
# NOTE: `causal` is currently unused here, but will be needed in the future
# to support models with different causality per-layer.
self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
@@ -156,6 +246,7 @@ class DFlashQwen3DecoderLayer(nn.Module):
vllm_config: VllmConfig,
*,
config: Qwen3Config,
layer_idx: int,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
@@ -165,6 +256,18 @@ class DFlashQwen3DecoderLayer(nn.Module):
set_default_rope_theta(config, default_theta=1000000)
attn_type = AttentionType.DECODER
# DFlash drafts store the sink-bias flag inside dflash_config; fall back
# to the top-level attribute used by other (e.g. MiMo) configs.
dflash_config = getattr(config, "dflash_config", None) or {}
add_swa_attention_sink_bias = dflash_config.get(
"attention_sink_bias",
getattr(config, "add_swa_attention_sink_bias", False),
)
# Resolve this layer's attention mode (full vs sliding window, causal vs
# non-causal) from the draft config.
sliding_window, causal = _resolve_layer_attention(config, layer_idx)
self.self_attn = DFlashQwen3Attention(
hidden_size=self.hidden_size,
num_heads=config.num_attention_heads,
@@ -172,6 +275,9 @@ class DFlashQwen3DecoderLayer(nn.Module):
num_kv_heads=config.num_key_value_heads,
rms_norm_eps=config.rms_norm_eps,
attention_bias=getattr(config, "attention_bias", False),
add_swa_attention_sink_bias=add_swa_attention_sink_bias,
sliding_window=sliding_window,
causal=causal,
head_dim=getattr(config, "head_dim", None),
cache_config=cache_config,
quant_config=quant_config,
@@ -243,11 +349,24 @@ class DFlashQwen3Model(nn.Module):
prefix=maybe_prefix(prefix, "embed_tokens"),
)
# Masked query slots are fed to the draft as `mask_token_id`. Most DFlash
# checkpoints will have the mask embedding in the vocabulary embedding table
# at that slot id. Some checkpoints (XiaomiMiMo/MiMo-V2.5-Pro-FP4-DFlash) ship
# with a separate mask embedding tensor to use instead. When present, we load it
# and substitute it for embed_tokens[mask_token_id] when computing embeddings.
self.mask_token_id = drafter_config.get("mask_token_id")
self.mask_embedding = nn.Parameter(
torch.zeros(self.config.hidden_size, dtype=vllm_config.model_config.dtype),
requires_grad=False,
)
self.has_separate_mask_embedding = False
self.layers = nn.ModuleList(
[
DFlashQwen3DecoderLayer(
current_vllm_config,
config=self.config,
layer_idx=layer_idx,
cache_config=current_vllm_config.cache_config,
quant_config=self.quant_config,
prefix=maybe_prefix(prefix, f"layers.{layer_idx + start_layer_id}"),
@@ -284,7 +403,12 @@ class DFlashQwen3Model(nn.Module):
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
embeds = self.embed_tokens(input_ids)
if self.has_separate_mask_embedding and self.mask_token_id is not None:
# Replace masked slots with the dedicated mask embedding.
is_mask = (input_ids == self.mask_token_id).unsqueeze(-1)
embeds = torch.where(is_mask, self.mask_embedding.to(embeds.dtype), embeds)
return embeds
def _build_fused_kv_buffers(self) -> None:
"""Build fused weight buffers for precompute_and_store_context_kv.
@@ -469,6 +593,8 @@ class DFlashQwen3Model(nn.Module):
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
tp_rank = get_tensor_model_parallel_rank()
tp_size = get_tensor_model_parallel_world_size()
for name, loaded_weight in weights:
if "midlayer." in name:
name = name.replace("midlayer.", "layers.0.")
@@ -476,6 +602,18 @@ class DFlashQwen3Model(nn.Module):
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
if "attention_sink_bias" in name:
if name not in params_dict:
continue
# Sink bias is per-head; shard it across TP ranks like the
# attention heads themselves.
param = params_dict[name]
heads_per_rank = loaded_weight.shape[0] // tp_size
head_start = tp_rank * heads_per_rank
narrow_weight = loaded_weight.narrow(0, head_start, heads_per_rank)
param.data.copy_(narrow_weight)
loaded_params.add(name)
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
@@ -495,7 +633,8 @@ class DFlashQwen3Model(nn.Module):
class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
nn.Module.__init__(self)
self.config = vllm_config.speculative_config.draft_model_config.hf_config
self.draft_model_config = vllm_config.speculative_config.draft_model_config
self.config = self.draft_model_config.hf_config
if getattr(self.config, "draft_vocab_size", None) is None:
self.config.draft_vocab_size = getattr(self.config, "vocab_size", None)
target_layer_num = vllm_config.model_config.get_num_layers(
@@ -589,7 +728,9 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
includes_embed_tokens = False
for name, loaded_weight in weights:
assert "mask_hidden" not in name, (
"DFlash should use mask_token_id to embed the padding hidden state"
"DFlash embeds masked slots via mask_token_id (optionally "
"overridden by a mask_embedding.pt file); it should not ship a "
"mask_hidden weight."
)
if "t2d" in name:
continue
@@ -603,6 +744,13 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
model_weights[name] = loaded_weight
process_eagle_weight(self, name)
# Route the separately-trained mask embedding (if shipped) through the
# standard weight loader alongside the rest of the draft weights.
mask_embedding = self._read_mask_embedding()
if mask_embedding is not None:
model_weights["model.mask_embedding"] = mask_embedding
self.model.has_separate_mask_embedding = True
skip_substrs = []
if not includes_draft_id_mapping:
skip_substrs.append("draft_id_to_target_id")
@@ -610,6 +758,8 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
skip_substrs.append("embed_tokens")
if not self.model.use_aux_hidden_state:
skip_substrs.append("fc.")
if not self.model.has_separate_mask_embedding:
skip_substrs.append("mask_embedding")
loader = AutoWeightsLoader(
self,
skip_prefixes=None,
@@ -617,3 +767,42 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
)
loader.load_weights(model_weights.items())
self.model._build_fused_kv_buffers()
def _read_mask_embedding(self) -> torch.Tensor | None:
"""Checks for an override mask embedding in `mask_embedding.pt` and returns it.
Some checkpoints ship a separately-trained mask embedding for the mask token,
which we use to overwrite the embedding for `mask_token_id`. This helper
checks for the file, loads the pytorch tensor, and returns the embedding to use.
Returns None if the override file is not present.
"""
mask_token_id = self.model.mask_token_id
if mask_token_id is None:
return None
MASK_EMBEDDING_FILENAME = "mask_embedding.pt"
data = get_hf_file_bytes(
MASK_EMBEDDING_FILENAME,
self.draft_model_config.model,
self.draft_model_config.revision,
)
if data is None:
return None
state = torch.load(io.BytesIO(data), weights_only=True)
if isinstance(state, dict):
if state.get("mask_token_id", mask_token_id) != mask_token_id:
raise ValueError(
f"{MASK_EMBEDDING_FILENAME} mask_token_id does not match "
f"dflash_config.mask_token_id ({mask_token_id}). "
f"Got {state.get('mask_token_id')}."
)
state = state["embedding"]
logger.info(
"Loaded DFlash mask embedding for mask_token_id %s from %s",
mask_token_id,
MASK_EMBEDDING_FILENAME,
)
return state.reshape(-1)
+33 -17
View File
@@ -252,6 +252,8 @@ class FlashAttentionMetadata:
causal: bool | torch.Tensor = True
sliding_window: tuple[int, int] | None = None
# PrefixLM bidirectional ranges for multimodal tokens.
# Shape: (num_seqs, max_ranges, 2) int32, [start, end] per range.
mm_prefix_range_tensor: torch.Tensor | None = None
@@ -284,6 +286,20 @@ def _get_sliding_window_configs(
return sliding_window_configs
def _maybe_symmetrize_window(
window: tuple[int, int] | None,
causal: bool | torch.Tensor,
) -> tuple[int, int] | None:
"""Make a causal sliding window ``(w, 0)`` symmetric ``(w, w)`` when attention
is non-causal, so bidirectional queries attend in both directions. Leaves
full-attention ``(-1, -1)`` and already-symmetric windows untouched.
"""
non_causal = isinstance(causal, torch.Tensor) or causal is False
if window is not None and window[0] >= 0 and window[1] == 0 and non_causal:
return (window[0], window[0])
return window
class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetadata]):
# FA3:
# Supports full cudagraphs for all cases.
@@ -487,7 +503,9 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad
cu_seqlens_q=cu_query_lens,
page_size=self.block_size,
causal=causal,
window_size=self.aot_sliding_window,
window_size=_maybe_symmetrize_window(
self.aot_sliding_window, causal
),
num_splits=max_num_splits,
)
return None
@@ -579,6 +597,13 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad
if isinstance(causal, torch.Tensor) and causal.dtype != torch.int32:
causal = causal.to(torch.int32)
# Symmetrize the spec's sliding_window for non-causal attention
group_sliding_window = getattr(self.kv_cache_spec, "sliding_window", None)
base_window = (
(-1, -1) if group_sliding_window is None else (group_sliding_window - 1, 0)
)
effective_sliding_window = _maybe_symmetrize_window(base_window, causal)
attn_metadata = FlashAttentionMetadata(
num_actual_tokens=num_actual_tokens,
max_query_len=max_query_len,
@@ -598,6 +623,7 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad
prefix_scheduler_metadata=prefix_scheduler_metadata,
max_num_splits=max_num_splits,
causal=causal,
sliding_window=effective_sliding_window,
)
# Compute mm_prefix range tensor if the batch contains
@@ -844,28 +870,18 @@ class FlashAttentionImpl(AttentionImpl):
)
return output
else:
window = (
attn_metadata.sliding_window
if attn_metadata.sliding_window is not None
else self.sliding_window
)
sliding_window_size: list[int] | None = (
list(self.sliding_window)
if self.sliding_window is not None
else None
list(window) if window is not None else None
)
causal = attn_metadata.causal
is_dynamic_causal = isinstance(causal, torch.Tensor)
# For non-causal (bidirectional) attention, make the
# sliding window symmetric so queries attend in both
# directions.
if (
sliding_window_size is not None
and sliding_window_size[1] == 0
and (is_dynamic_causal or causal is False)
):
sliding_window_size = [
sliding_window_size[0],
sliding_window_size[0],
]
mm_prefix_ranges = attn_metadata.mm_prefix_range_tensor
mm_mask_mod = None
mm_aux = None