[Spec Decode] Support SWA + DFlash for MiMo (#46104)
This commit is contained in:
@@ -1408,7 +1408,7 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
# [DFlash]
|
||||
"DFlashDraftModel": _HfExamplesInfo(
|
||||
"Qwen/Qwen3.5-4B",
|
||||
speculative_model="z-lab/Qwen3.5-4B-DFlash",
|
||||
speculative_model="z-lab/Qwen3-4B-DFlash-b16",
|
||||
use_original_num_layers=True, # Need all layers since DFlash has >1 layer,
|
||||
max_model_len=8192, # Reduce max len to ensure test runs in low-VRAM CI env
|
||||
max_num_seqs=32,
|
||||
|
||||
@@ -53,7 +53,12 @@ from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
from .interfaces import MixtureOfExperts, SupportsPP
|
||||
from .interfaces import (
|
||||
EagleModelMixin,
|
||||
MixtureOfExperts,
|
||||
SupportsEagle3,
|
||||
SupportsPP,
|
||||
)
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
@@ -539,7 +544,7 @@ def _shard_fp8_qkv_proj(
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
class MiMoV2Model(nn.Module):
|
||||
class MiMoV2Model(nn.Module, EagleModelMixin):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
@@ -602,10 +607,16 @@ class MiMoV2Model(nn.Module):
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
residual = intermediate_tensors["residual"]
|
||||
|
||||
aux_hidden_states = self._maybe_add_hidden_state(
|
||||
[], self.start_layer, hidden_states, residual
|
||||
)
|
||||
for idx, layer in enumerate(
|
||||
islice(self.layers, self.start_layer, self.end_layer)
|
||||
):
|
||||
hidden_states, residual = layer(positions, hidden_states, residual)
|
||||
self._maybe_add_hidden_state(
|
||||
aux_hidden_states, idx + 1, hidden_states, residual
|
||||
)
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
return IntermediateTensors(
|
||||
@@ -614,6 +625,8 @@ class MiMoV2Model(nn.Module):
|
||||
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
|
||||
if len(aux_hidden_states) > 0:
|
||||
return hidden_states, aux_hidden_states
|
||||
return hidden_states
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
@@ -822,7 +835,7 @@ class MiMoV2Model(nn.Module):
|
||||
return True
|
||||
|
||||
|
||||
class MiMoV2FlashForCausalLM(nn.Module, SupportsPP, MixtureOfExperts):
|
||||
class MiMoV2FlashForCausalLM(nn.Module, SupportsPP, MixtureOfExperts, SupportsEagle3):
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import io
|
||||
from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
@@ -11,7 +12,10 @@ from transformers import Qwen3Config
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -33,6 +37,7 @@ from vllm.model_executor.model_loader.weight_utils import (
|
||||
)
|
||||
from vllm.multimodal.inputs import NestedTensors
|
||||
from vllm.transformers_utils.config import set_default_rope_theta
|
||||
from vllm.transformers_utils.repo_utils import get_hf_file_bytes
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
|
||||
from .qwen2 import Qwen2MLP as Qwen3MLP
|
||||
@@ -47,6 +52,76 @@ from .utils import (
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _resolve_layer_attention(
|
||||
config: Qwen3Config, layer_idx: int
|
||||
) -> tuple[int | None, bool]:
|
||||
"""Resolve ``(sliding_window, causal)`` for one DFlash draft layer.
|
||||
|
||||
+----------------------+-------------------------+--------------------------------+
|
||||
| Config | ``layer_type`` | *``causal`` |
|
||||
+======================+=========================+================================+
|
||||
| ``layer_types`` | SWA if ``use_swa`` | True if ``layer_types[i]=SWA`` |
|
||||
| | else ``layer_types[i]`` | else False |
|
||||
+----------------------+-------------------------+--------------------------------+
|
||||
| ``layer_types=None`` | SWA | False |
|
||||
| + ``use_swa=True`` | | |
|
||||
+----------------------+-------------------------+--------------------------------+
|
||||
| ``layer_types=None`` | Full | False |
|
||||
| + ``use_swa=False`` | | |
|
||||
+----------------------+-------------------------+--------------------------------+
|
||||
* If ``dflash_config.causal`` is set, its value overrides ``causal`` for all layers.
|
||||
|
||||
This is to support a varied ecosystem of checkpoints, including:
|
||||
- XiaomiMiMo/MiMo-V2.5-Pro-FP4-DFlash (sets "use_swa", assumes non-causal)
|
||||
- z-lab/gemma-4-31B-it-DFlash (has mixed layer types, assumes causal only for SWA)
|
||||
- z-lab/Qwen3.5-9B-DFlash ("standard" DFlash, all full attn, assumes non-causal)
|
||||
"""
|
||||
dflash_config = getattr(config, "dflash_config", None) or {}
|
||||
layer_types = getattr(config, "layer_types", None)
|
||||
use_swa = dflash_config.get("use_swa", False)
|
||||
config_causal = dflash_config.get("causal", None)
|
||||
|
||||
SLIDING_ATTENTION = "sliding_attention"
|
||||
any_sliding = False
|
||||
if layer_types is not None:
|
||||
num_sliding = sum(lt == SLIDING_ATTENTION for lt in layer_types)
|
||||
any_sliding = num_sliding > 0
|
||||
all_sliding = num_sliding == len(layer_types)
|
||||
if any_sliding and not all_sliding:
|
||||
# Mixed sliding/full attention needs per-layer causal metadata and
|
||||
# multiple KV-cache groups, which DFlash does not yet support.
|
||||
raise NotImplementedError(
|
||||
"DFlash does not yet support mixed sliding/full attention via "
|
||||
"layer_types; see "
|
||||
"https://github.com/vllm-project/vllm/issues/40898."
|
||||
)
|
||||
|
||||
default_causal = False
|
||||
if layer_types is None or (use_swa and not any_sliding):
|
||||
# An absent ``layer_types`` (or the all-"full_attention" one that may
|
||||
# be synthesized when the checkpoint omits it) must not override
|
||||
# ``dflash_config.use_swa``, which forces SWA on every layer.
|
||||
is_sliding = use_swa
|
||||
else:
|
||||
is_sliding = layer_types[layer_idx] == SLIDING_ATTENTION
|
||||
# Full-attention layers default non-causal; SWA layers default causal.
|
||||
default_causal = is_sliding
|
||||
|
||||
sliding_window = None
|
||||
if is_sliding:
|
||||
sliding_window = dflash_config.get(
|
||||
"swa_window_size", getattr(config, "sliding_window", None)
|
||||
)
|
||||
if sliding_window is None:
|
||||
raise ValueError(
|
||||
"DFlash sliding attention requires a window size configured in "
|
||||
"dflash_config.swa_window_size or the top-level sliding_window."
|
||||
)
|
||||
|
||||
causal = config_causal if config_causal is not None else default_causal
|
||||
return sliding_window, causal
|
||||
|
||||
|
||||
class DFlashQwen3Attention(nn.Module):
|
||||
"""Attention for DFlash speculative decoding.
|
||||
|
||||
@@ -64,6 +139,9 @@ class DFlashQwen3Attention(nn.Module):
|
||||
head_dim: int | None = None,
|
||||
rms_norm_eps: float = 1e-06,
|
||||
attention_bias: bool = False,
|
||||
add_swa_attention_sink_bias: bool = False,
|
||||
sliding_window: int | None = None,
|
||||
causal: bool = False,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
@@ -109,6 +187,14 @@ class DFlashQwen3Attention(nn.Module):
|
||||
max_position=max_position,
|
||||
rope_parameters=rope_parameters,
|
||||
)
|
||||
|
||||
self.attention_sink_bias = (
|
||||
torch.nn.Parameter(torch.empty(self.num_heads), requires_grad=False)
|
||||
if add_swa_attention_sink_bias
|
||||
else None
|
||||
)
|
||||
|
||||
self.sliding_window = sliding_window
|
||||
self.attn = Attention(
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
@@ -116,9 +202,13 @@ class DFlashQwen3Attention(nn.Module):
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
per_layer_sliding_window=sliding_window,
|
||||
prefix=f"{prefix}.attn",
|
||||
attn_type=attn_type,
|
||||
sinks=self.attention_sink_bias,
|
||||
)
|
||||
# NOTE: `causal` is currently unused here, but will be needed in the future
|
||||
# to support models with different causality per-layer.
|
||||
self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
|
||||
self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
|
||||
|
||||
@@ -156,6 +246,7 @@ class DFlashQwen3DecoderLayer(nn.Module):
|
||||
vllm_config: VllmConfig,
|
||||
*,
|
||||
config: Qwen3Config,
|
||||
layer_idx: int,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
@@ -165,6 +256,18 @@ class DFlashQwen3DecoderLayer(nn.Module):
|
||||
set_default_rope_theta(config, default_theta=1000000)
|
||||
attn_type = AttentionType.DECODER
|
||||
|
||||
# DFlash drafts store the sink-bias flag inside dflash_config; fall back
|
||||
# to the top-level attribute used by other (e.g. MiMo) configs.
|
||||
dflash_config = getattr(config, "dflash_config", None) or {}
|
||||
add_swa_attention_sink_bias = dflash_config.get(
|
||||
"attention_sink_bias",
|
||||
getattr(config, "add_swa_attention_sink_bias", False),
|
||||
)
|
||||
|
||||
# Resolve this layer's attention mode (full vs sliding window, causal vs
|
||||
# non-causal) from the draft config.
|
||||
sliding_window, causal = _resolve_layer_attention(config, layer_idx)
|
||||
|
||||
self.self_attn = DFlashQwen3Attention(
|
||||
hidden_size=self.hidden_size,
|
||||
num_heads=config.num_attention_heads,
|
||||
@@ -172,6 +275,9 @@ class DFlashQwen3DecoderLayer(nn.Module):
|
||||
num_kv_heads=config.num_key_value_heads,
|
||||
rms_norm_eps=config.rms_norm_eps,
|
||||
attention_bias=getattr(config, "attention_bias", False),
|
||||
add_swa_attention_sink_bias=add_swa_attention_sink_bias,
|
||||
sliding_window=sliding_window,
|
||||
causal=causal,
|
||||
head_dim=getattr(config, "head_dim", None),
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
@@ -243,11 +349,24 @@ class DFlashQwen3Model(nn.Module):
|
||||
prefix=maybe_prefix(prefix, "embed_tokens"),
|
||||
)
|
||||
|
||||
# Masked query slots are fed to the draft as `mask_token_id`. Most DFlash
|
||||
# checkpoints will have the mask embedding in the vocabulary embedding table
|
||||
# at that slot id. Some checkpoints (XiaomiMiMo/MiMo-V2.5-Pro-FP4-DFlash) ship
|
||||
# with a separate mask embedding tensor to use instead. When present, we load it
|
||||
# and substitute it for embed_tokens[mask_token_id] when computing embeddings.
|
||||
self.mask_token_id = drafter_config.get("mask_token_id")
|
||||
self.mask_embedding = nn.Parameter(
|
||||
torch.zeros(self.config.hidden_size, dtype=vllm_config.model_config.dtype),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.has_separate_mask_embedding = False
|
||||
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
DFlashQwen3DecoderLayer(
|
||||
current_vllm_config,
|
||||
config=self.config,
|
||||
layer_idx=layer_idx,
|
||||
cache_config=current_vllm_config.cache_config,
|
||||
quant_config=self.quant_config,
|
||||
prefix=maybe_prefix(prefix, f"layers.{layer_idx + start_layer_id}"),
|
||||
@@ -284,7 +403,12 @@ class DFlashQwen3Model(nn.Module):
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
embeds = self.embed_tokens(input_ids)
|
||||
if self.has_separate_mask_embedding and self.mask_token_id is not None:
|
||||
# Replace masked slots with the dedicated mask embedding.
|
||||
is_mask = (input_ids == self.mask_token_id).unsqueeze(-1)
|
||||
embeds = torch.where(is_mask, self.mask_embedding.to(embeds.dtype), embeds)
|
||||
return embeds
|
||||
|
||||
def _build_fused_kv_buffers(self) -> None:
|
||||
"""Build fused weight buffers for precompute_and_store_context_kv.
|
||||
@@ -469,6 +593,8 @@ class DFlashQwen3Model(nn.Module):
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
for name, loaded_weight in weights:
|
||||
if "midlayer." in name:
|
||||
name = name.replace("midlayer.", "layers.0.")
|
||||
@@ -476,6 +602,18 @@ class DFlashQwen3Model(nn.Module):
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
continue
|
||||
if "attention_sink_bias" in name:
|
||||
if name not in params_dict:
|
||||
continue
|
||||
# Sink bias is per-head; shard it across TP ranks like the
|
||||
# attention heads themselves.
|
||||
param = params_dict[name]
|
||||
heads_per_rank = loaded_weight.shape[0] // tp_size
|
||||
head_start = tp_rank * heads_per_rank
|
||||
narrow_weight = loaded_weight.narrow(0, head_start, heads_per_rank)
|
||||
param.data.copy_(narrow_weight)
|
||||
loaded_params.add(name)
|
||||
continue
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
@@ -495,7 +633,8 @@ class DFlashQwen3Model(nn.Module):
|
||||
class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
nn.Module.__init__(self)
|
||||
self.config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
self.draft_model_config = vllm_config.speculative_config.draft_model_config
|
||||
self.config = self.draft_model_config.hf_config
|
||||
if getattr(self.config, "draft_vocab_size", None) is None:
|
||||
self.config.draft_vocab_size = getattr(self.config, "vocab_size", None)
|
||||
target_layer_num = vllm_config.model_config.get_num_layers(
|
||||
@@ -589,7 +728,9 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
|
||||
includes_embed_tokens = False
|
||||
for name, loaded_weight in weights:
|
||||
assert "mask_hidden" not in name, (
|
||||
"DFlash should use mask_token_id to embed the padding hidden state"
|
||||
"DFlash embeds masked slots via mask_token_id (optionally "
|
||||
"overridden by a mask_embedding.pt file); it should not ship a "
|
||||
"mask_hidden weight."
|
||||
)
|
||||
if "t2d" in name:
|
||||
continue
|
||||
@@ -603,6 +744,13 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
|
||||
model_weights[name] = loaded_weight
|
||||
process_eagle_weight(self, name)
|
||||
|
||||
# Route the separately-trained mask embedding (if shipped) through the
|
||||
# standard weight loader alongside the rest of the draft weights.
|
||||
mask_embedding = self._read_mask_embedding()
|
||||
if mask_embedding is not None:
|
||||
model_weights["model.mask_embedding"] = mask_embedding
|
||||
self.model.has_separate_mask_embedding = True
|
||||
|
||||
skip_substrs = []
|
||||
if not includes_draft_id_mapping:
|
||||
skip_substrs.append("draft_id_to_target_id")
|
||||
@@ -610,6 +758,8 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
|
||||
skip_substrs.append("embed_tokens")
|
||||
if not self.model.use_aux_hidden_state:
|
||||
skip_substrs.append("fc.")
|
||||
if not self.model.has_separate_mask_embedding:
|
||||
skip_substrs.append("mask_embedding")
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=None,
|
||||
@@ -617,3 +767,42 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
|
||||
)
|
||||
loader.load_weights(model_weights.items())
|
||||
self.model._build_fused_kv_buffers()
|
||||
|
||||
def _read_mask_embedding(self) -> torch.Tensor | None:
|
||||
"""Checks for an override mask embedding in `mask_embedding.pt` and returns it.
|
||||
|
||||
Some checkpoints ship a separately-trained mask embedding for the mask token,
|
||||
which we use to overwrite the embedding for `mask_token_id`. This helper
|
||||
checks for the file, loads the pytorch tensor, and returns the embedding to use.
|
||||
|
||||
Returns None if the override file is not present.
|
||||
"""
|
||||
mask_token_id = self.model.mask_token_id
|
||||
if mask_token_id is None:
|
||||
return None
|
||||
|
||||
MASK_EMBEDDING_FILENAME = "mask_embedding.pt"
|
||||
data = get_hf_file_bytes(
|
||||
MASK_EMBEDDING_FILENAME,
|
||||
self.draft_model_config.model,
|
||||
self.draft_model_config.revision,
|
||||
)
|
||||
if data is None:
|
||||
return None
|
||||
|
||||
state = torch.load(io.BytesIO(data), weights_only=True)
|
||||
if isinstance(state, dict):
|
||||
if state.get("mask_token_id", mask_token_id) != mask_token_id:
|
||||
raise ValueError(
|
||||
f"{MASK_EMBEDDING_FILENAME} mask_token_id does not match "
|
||||
f"dflash_config.mask_token_id ({mask_token_id}). "
|
||||
f"Got {state.get('mask_token_id')}."
|
||||
)
|
||||
state = state["embedding"]
|
||||
|
||||
logger.info(
|
||||
"Loaded DFlash mask embedding for mask_token_id %s from %s",
|
||||
mask_token_id,
|
||||
MASK_EMBEDDING_FILENAME,
|
||||
)
|
||||
return state.reshape(-1)
|
||||
|
||||
@@ -252,6 +252,8 @@ class FlashAttentionMetadata:
|
||||
|
||||
causal: bool | torch.Tensor = True
|
||||
|
||||
sliding_window: tuple[int, int] | None = None
|
||||
|
||||
# PrefixLM bidirectional ranges for multimodal tokens.
|
||||
# Shape: (num_seqs, max_ranges, 2) int32, [start, end] per range.
|
||||
mm_prefix_range_tensor: torch.Tensor | None = None
|
||||
@@ -284,6 +286,20 @@ def _get_sliding_window_configs(
|
||||
return sliding_window_configs
|
||||
|
||||
|
||||
def _maybe_symmetrize_window(
|
||||
window: tuple[int, int] | None,
|
||||
causal: bool | torch.Tensor,
|
||||
) -> tuple[int, int] | None:
|
||||
"""Make a causal sliding window ``(w, 0)`` symmetric ``(w, w)`` when attention
|
||||
is non-causal, so bidirectional queries attend in both directions. Leaves
|
||||
full-attention ``(-1, -1)`` and already-symmetric windows untouched.
|
||||
"""
|
||||
non_causal = isinstance(causal, torch.Tensor) or causal is False
|
||||
if window is not None and window[0] >= 0 and window[1] == 0 and non_causal:
|
||||
return (window[0], window[0])
|
||||
return window
|
||||
|
||||
|
||||
class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetadata]):
|
||||
# FA3:
|
||||
# Supports full cudagraphs for all cases.
|
||||
@@ -487,7 +503,9 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad
|
||||
cu_seqlens_q=cu_query_lens,
|
||||
page_size=self.block_size,
|
||||
causal=causal,
|
||||
window_size=self.aot_sliding_window,
|
||||
window_size=_maybe_symmetrize_window(
|
||||
self.aot_sliding_window, causal
|
||||
),
|
||||
num_splits=max_num_splits,
|
||||
)
|
||||
return None
|
||||
@@ -579,6 +597,13 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad
|
||||
if isinstance(causal, torch.Tensor) and causal.dtype != torch.int32:
|
||||
causal = causal.to(torch.int32)
|
||||
|
||||
# Symmetrize the spec's sliding_window for non-causal attention
|
||||
group_sliding_window = getattr(self.kv_cache_spec, "sliding_window", None)
|
||||
base_window = (
|
||||
(-1, -1) if group_sliding_window is None else (group_sliding_window - 1, 0)
|
||||
)
|
||||
effective_sliding_window = _maybe_symmetrize_window(base_window, causal)
|
||||
|
||||
attn_metadata = FlashAttentionMetadata(
|
||||
num_actual_tokens=num_actual_tokens,
|
||||
max_query_len=max_query_len,
|
||||
@@ -598,6 +623,7 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad
|
||||
prefix_scheduler_metadata=prefix_scheduler_metadata,
|
||||
max_num_splits=max_num_splits,
|
||||
causal=causal,
|
||||
sliding_window=effective_sliding_window,
|
||||
)
|
||||
|
||||
# Compute mm_prefix range tensor if the batch contains
|
||||
@@ -844,28 +870,18 @@ class FlashAttentionImpl(AttentionImpl):
|
||||
)
|
||||
return output
|
||||
else:
|
||||
window = (
|
||||
attn_metadata.sliding_window
|
||||
if attn_metadata.sliding_window is not None
|
||||
else self.sliding_window
|
||||
)
|
||||
sliding_window_size: list[int] | None = (
|
||||
list(self.sliding_window)
|
||||
if self.sliding_window is not None
|
||||
else None
|
||||
list(window) if window is not None else None
|
||||
)
|
||||
|
||||
causal = attn_metadata.causal
|
||||
is_dynamic_causal = isinstance(causal, torch.Tensor)
|
||||
|
||||
# For non-causal (bidirectional) attention, make the
|
||||
# sliding window symmetric so queries attend in both
|
||||
# directions.
|
||||
if (
|
||||
sliding_window_size is not None
|
||||
and sliding_window_size[1] == 0
|
||||
and (is_dynamic_causal or causal is False)
|
||||
):
|
||||
sliding_window_size = [
|
||||
sliding_window_size[0],
|
||||
sliding_window_size[0],
|
||||
]
|
||||
|
||||
mm_prefix_ranges = attn_metadata.mm_prefix_range_tensor
|
||||
mm_mask_mod = None
|
||||
mm_aux = None
|
||||
|
||||
Reference in New Issue
Block a user