diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index f1ba46b707..70956f5092 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -472,6 +472,7 @@ th { | `Qwen3MoeForCausalLM` | Qwen3MoE | `Qwen/Qwen3-30B-A3B`, etc. | ✅︎ | ✅︎ | | `Qwen3NextForCausalLM` | Qwen3NextMoE | `Qwen/Qwen3-Next-80B-A3B-Instruct`, etc. | ✅︎ | ✅︎ | | `RWForCausalLM` | Falcon RW | `tiiuae/falcon-40b`, etc. | | ✅︎ | +| `Rnj1ForCausalLM` | Rnj1 | `EssentialAI/rnj-1-instruct`, etc. | | | | `SarvamMoEForCausalLM` | Sarvam 2 | `sarvamai/sarvam2-30b-a3b`, etc. | ✅︎ | ✅︎ | | `SarvamMLAForCausalLM` | Sarvam 2 | `sarvamai/sarvam2-105b-a9b`, etc. | | ✅︎ | | `SeedOssForCausalLM` | SeedOss | `ByteDance-Seed/Seed-OSS-36B-Instruct`, etc. | ✅︎ | ✅︎ | diff --git a/tests/models/registry.py b/tests/models/registry.py index ea1f019056..4c418ae4ee 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -518,6 +518,10 @@ _TEXT_GENERATION_EXAMPLE_MODELS = { extras={"tiny-random": "tiny-random/qwen3-next-moe"}, min_transformers_version="4.56.3", ), + "Rnj1ForCausalLM": _HfExamplesInfo( + "EssentialAI/rnj-1-instruct", + is_available_online=False, + ), "RWForCausalLM": _HfExamplesInfo("tiiuae/falcon-40b"), "SarvamMoEForCausalLM": _HfExamplesInfo( "sarvamai/sarvam-30b", diff --git a/vllm/model_executor/layers/attention/attention.py b/vllm/model_executor/layers/attention/attention.py index 9d2e29d02d..d229e32be7 100644 --- a/vllm/model_executor/layers/attention/attention.py +++ b/vllm/model_executor/layers/attention/attention.py @@ -336,6 +336,12 @@ class Attention(nn.Module, AttentionLayerBase): ) cache_config.enable_prefix_caching = False + if extra_impl_args.get("chunk_lookback", -1) > -1: + assert self.attn_backend.get_name() == "TRITON_ATTN", ( + f"Chunked attention with lookback requires the Triton backend, " + f"but got {self.attn_backend.get_name()}." + ) + impl_cls = self.attn_backend.get_impl_cls() self.impl = impl_cls( # type: ignore[assignment] # impl_cls always returns an AttentionImpl subclass num_heads, diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 4ba774a9fe..baac4a6c66 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -110,6 +110,7 @@ _TEXT_GENERATION_MODELS = { "GemmaForCausalLM": ("gemma", "GemmaForCausalLM"), "Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"), "Gemma3ForCausalLM": ("gemma3", "Gemma3ForCausalLM"), + "Rnj1ForCausalLM": ("rnj1", "Rnj1ForCausalLM"), "Gemma3nForCausalLM": ("gemma3n", "Gemma3nForCausalLM"), "Gemma4ForCausalLM": ("gemma4", "Gemma4ForCausalLM"), "Qwen3NextForCausalLM": ("qwen3_next", "Qwen3NextForCausalLM"), diff --git a/vllm/model_executor/models/rnj1.py b/vllm/model_executor/models/rnj1.py new file mode 100644 index 0000000000..f83577b7a3 --- /dev/null +++ b/vllm/model_executor/models/rnj1.py @@ -0,0 +1,470 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# +# RNJ-1 model: Gemma3-based architecture with chunked (block-local) attention. +# Chunked attention restricts local layers to attend within aligned blocks, +# with lookback to one previous block. +from collections.abc import Iterable +from itertools import islice + +import torch +from torch import nn +from transformers import Gemma3TextConfig + +from vllm.compilation.decorators import support_torch_compile +from vllm.config import CacheConfig, VllmConfig +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size +from vllm.logger import init_logger +from vllm.model_executor.layers.activation import GeluAndMul +from vllm.model_executor.layers.attention import Attention +from vllm.model_executor.layers.layernorm import GemmaRMSNorm +from vllm.model_executor.layers.linear import ( + MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear, +) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, + VocabParallelEmbedding, +) +from vllm.model_executor.model_loader.weight_utils import ( + default_weight_loader, + maybe_remap_kv_scale_name, +) +from vllm.sequence import IntermediateTensors +from vllm.v1.attention.backend import AttentionType + +from .interfaces import SupportsLoRA, SupportsPP +from .utils import ( + AutoWeightsLoader, + extract_layer_index, + is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, + make_layers, + maybe_prefix, +) + +logger = init_logger(__name__) + + +class Rnj1MLP(nn.Module): + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_activation: str, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + self.gate_up_proj = MergedColumnParallelLinear( + hidden_size, + [intermediate_size] * 2, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.gate_up_proj", + ) + self.down_proj = RowParallelLinear( + intermediate_size, + hidden_size, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.down_proj", + ) + if hidden_activation != "gelu_pytorch_tanh": + raise ValueError( + "RNJ-1 uses `gelu_pytorch_tanh` as the hidden activation " + "function. Please set `hidden_act` and `hidden_activation` to " + "`gelu_pytorch_tanh`." + ) + self.act_fn = GeluAndMul(approximate="tanh") + + def forward(self, x: torch.Tensor) -> torch.Tensor: + gate_up, _ = self.gate_up_proj(x) + x = self.act_fn(gate_up) + x, _ = self.down_proj(x) + return x + + +class Rnj1Attention(nn.Module): + def __init__( + self, + config: Gemma3TextConfig, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + head_dim: int, + max_position_embeddings: int, + cache_config: CacheConfig | None = None, + quant_config: QuantizationConfig | None = None, + attn_logits_soft_cap: float | None = None, + prefix: str = "", + ) -> None: + super().__init__() + self.config = config + self.hidden_size = hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = num_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = num_kv_heads + if self.total_num_kv_heads >= tp_size: + assert self.total_num_kv_heads % tp_size == 0 + else: + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.head_dim = head_dim + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = config.query_pre_attn_scalar**-0.5 + + self.qkv_proj = QKVParallelLinear( + hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=config.attention_bias, + quant_config=quant_config, + prefix=f"{prefix}.qkv_proj", + ) + self.o_proj = RowParallelLinear( + self.total_num_heads * self.head_dim, + hidden_size, + bias=config.attention_bias, + quant_config=quant_config, + prefix=f"{prefix}.o_proj", + ) + + self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps) + self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps) + + layer_idx = extract_layer_index(prefix) + layer_type = config.layer_types[layer_idx] + self.is_chunked = layer_type == "chunked_attention" + self.chunk_lookback = 1 if self.is_chunked else -1 + sliding_window = config.sliding_window if self.is_chunked else None + + # Initialize the rotary embedding. + # Expects v5-style rope_parameters keyed by layer type. + if layer_type in config.rope_parameters: + rope_parameters = config.rope_parameters[layer_type] + else: + rope_parameters = config.rope_parameters + + self.rotary_emb = get_rope( + self.head_dim, + max_position=max_position_embeddings, + rope_parameters=rope_parameters, + is_neox_style=True, + ) + + self.attn = Attention( + self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config, + attn_type=AttentionType.DECODER, + logits_soft_cap=attn_logits_soft_cap, + per_layer_sliding_window=sliding_window, + chunk_lookback=self.chunk_lookback, + prefix=f"{prefix}.attn", + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + **kwargs, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + + q = q.unflatten(-1, (self.num_heads, self.head_dim)) + q = self.q_norm(q) + q = q.flatten(-2, -1) + k = k.unflatten(-1, (self.num_kv_heads, self.head_dim)) + k = self.k_norm(k) + k = k.flatten(-2, -1) + + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v) + output, _ = self.o_proj(attn_output) + return output + + +class Rnj1DecoderLayer(nn.Module): + def __init__( + self, + config: Gemma3TextConfig, + cache_config: CacheConfig | None = None, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + self.self_attn = Rnj1Attention( + config=config, + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + num_kv_heads=config.num_key_value_heads, + head_dim=config.head_dim, + max_position_embeddings=config.max_position_embeddings, + cache_config=cache_config, + quant_config=quant_config, + attn_logits_soft_cap=None, + prefix=f"{prefix}.self_attn", + ) + self.hidden_size = config.hidden_size + self.mlp = Rnj1MLP( + hidden_size=self.hidden_size, + intermediate_size=config.intermediate_size, + hidden_activation=config.hidden_activation, + quant_config=quant_config, + prefix=f"{prefix}.mlp", + ) + self.input_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = GemmaRMSNorm( + config.hidden_size, eps=config.rms_norm_eps + ) + self.pre_feedforward_layernorm = GemmaRMSNorm( + config.hidden_size, eps=config.rms_norm_eps + ) + self.post_feedforward_layernorm = GemmaRMSNorm( + config.hidden_size, eps=config.rms_norm_eps + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + residual: torch.Tensor | None, + **kwargs, + ) -> tuple[torch.Tensor, torch.Tensor]: + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm(hidden_states, residual) + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + **kwargs, + ) + hidden_states = self.post_attention_layernorm(hidden_states) + + hidden_states, residual = self.pre_feedforward_layernorm( + hidden_states, residual + ) + hidden_states = self.mlp(hidden_states) + hidden_states = self.post_feedforward_layernorm(hidden_states) + return hidden_states, residual + + +@support_torch_compile +class Rnj1Model(nn.Module): + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + super().__init__() + config = vllm_config.model_config.hf_config + cache_config = vllm_config.cache_config + quant_config = vllm_config.quant_config + self.config = config + self.quant_config = quant_config + + self.embed_tokens = VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + quant_config=quant_config, + prefix=f"{prefix}.embed_tokens", + ) + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: Rnj1DecoderLayer( + config, cache_config, quant_config, prefix=prefix + ), + prefix=f"{prefix}.layers", + ) + self.norm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + normalizer = self.config.hidden_size**0.5 + self.register_buffer("normalizer", torch.tensor(normalizer), persistent=False) + self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size + ) + + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.embed_tokens(input_ids) * self.normalizer + + def forward( + self, + input_ids: torch.Tensor | None, + positions: torch.Tensor, + intermediate_tensors: IntermediateTensors | None, + inputs_embeds: torch.Tensor | None = None, + **kwargs, + ) -> torch.Tensor | IntermediateTensors: + if get_pp_group().is_first_rank: + if inputs_embeds is not None: + hidden_states = inputs_embeds + else: + hidden_states = self.embed_input_ids(input_ids) + residual = None + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + for layer in islice(self.layers, self.start_layer, self.end_layer): + hidden_states, residual = layer( + positions, + hidden_states, + residual, + **kwargs, + ) + if not get_pp_group().is_last_rank: + return IntermediateTensors( + {"hidden_states": hidden_states, "residual": residual} + ) + hidden_states, _ = self.norm(hidden_states, residual) + return hidden_states + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ("gate_up_proj", "gate_proj", 0), + ("gate_up_proj", "up_proj", 1), + ] + params_dict = dict(self.named_parameters()) + loaded_params: set[str] = set() + for name, loaded_weight in weights: + if ( + self.quant_config + and self.quant_config.get_name() == "gguf" + and name.endswith("norm.weight") + ): + loaded_weight -= 1 + + if self.quant_config is not None and ( + scale_name := self.quant_config.get_cache_scale(name) + ): + param = params_dict[scale_name] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + loaded_weight = loaded_weight[0] + weight_loader(param, loaded_weight) + loaded_params.add(scale_name) + continue + + if name.endswith((".k_scale", ".v_scale", ".q_scale", ".prob_scale")): + remapped_name = maybe_remap_kv_scale_name(name, params_dict) + if remapped_name is not None and remapped_name in params_dict: + param = params_dict[remapped_name] + weight_loader = getattr( + param, "weight_loader", default_weight_loader + ) + weight_loader(param, loaded_weight) + loaded_params.add(remapped_name) + continue + + for param_name, shard_name, shard_id in stacked_params_mapping: + if shard_name not in name: + continue + name = name.replace(shard_name, param_name) + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + if name.endswith(".bias") and name not in params_dict: + continue + name = maybe_remap_kv_scale_name(name, params_dict) + if name is None: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader(param, loaded_weight) + loaded_params.add(name) + + return loaded_params + + +class Rnj1ForCausalLM(nn.Module, SupportsLoRA, SupportsPP): + packed_modules_mapping = { + "qkv_proj": [ + "q_proj", + "k_proj", + "v_proj", + ], + "gate_up_proj": [ + "gate_proj", + "up_proj", + ], + } + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + config = vllm_config.model_config.hf_config + quant_config = vllm_config.quant_config + + super().__init__() + self.config = config + self.quant_config = quant_config + self.model = Rnj1Model( + vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") + ) + + self.lm_head = ParallelLMHead( + config.vocab_size, + config.hidden_size, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "lm_head"), + ) + if config.tie_word_embeddings: + self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens) + + self.logits_processor = LogitsProcessor( + config.vocab_size, soft_cap=config.final_logit_softcapping + ) + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors + ) + + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.model.embed_input_ids(input_ids) + + def forward( + self, + input_ids: torch.Tensor | None, + positions: torch.Tensor, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + **kwargs, + ) -> torch.Tensor | IntermediateTensors: + hidden_states = self.model( + input_ids, positions, intermediate_tensors, inputs_embeds, **kwargs + ) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + ) -> torch.Tensor | None: + logits = self.logits_processor(self.lm_head, hidden_states) + return logits + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + loader = AutoWeightsLoader( + self, + skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None), + ) + return loader.load_weights(weights) diff --git a/vllm/v1/attention/backends/triton_attn.py b/vllm/v1/attention/backends/triton_attn.py index 76cae14aed..4739d48e87 100644 --- a/vllm/v1/attention/backends/triton_attn.py +++ b/vllm/v1/attention/backends/triton_attn.py @@ -458,6 +458,7 @@ class TritonAttentionImpl(AttentionImpl): kv_sharing_target_layer_name: int | None = None, sinks: torch.Tensor | None = None, use_alibi_sqrt: bool = False, + chunk_lookback: int = -1, ) -> None: self.num_heads = num_heads self.head_size = head_size @@ -492,6 +493,7 @@ class TritonAttentionImpl(AttentionImpl): f"num_heads: {num_heads}." ) self.use_alibi_sqrt = use_alibi_sqrt + self.chunk_lookback = chunk_lookback self.supports_quant_query_input = current_platform.is_cuda() self._kv_quant_mode = get_kv_quant_mode(kv_cache_dtype) @@ -631,6 +633,7 @@ class TritonAttentionImpl(AttentionImpl): kv_quant_mode=self._kv_quant_mode, k_scale_cache=k_scale_cache, v_scale_cache=v_scale_cache, + chunk_lookback=self.chunk_lookback, ) return output diff --git a/vllm/v1/attention/ops/triton_unified_attention.py b/vllm/v1/attention/ops/triton_unified_attention.py index 150f022f84..285c7b263b 100644 --- a/vllm/v1/attention/ops/triton_unified_attention.py +++ b/vllm/v1/attention/ops/triton_unified_attention.py @@ -177,6 +177,8 @@ def kernel_unified_attention_2d( stride_vs_blk=0, stride_vs_slot=0, stride_vs_head=0, + CHUNK_LOOKBACK: tl.constexpr = -1, + CHUNK_SIZE: tl.constexpr = -1, ): q_block_global_idx = tl.program_id(0) kv_head_idx = tl.program_id(1) @@ -293,7 +295,11 @@ def kernel_unified_attention_2d( # where q_abs = context_len + q # The union of allowed key positions for this Q-block is: # [context_len + qpos_lo - SLIDING_WINDOW + 1, context_len + qpos_hi] - first_allowed_key = context_len + qpos_lo - SLIDING_WINDOW + 1 + q_abs = context_len + qpos_lo + if CHUNK_LOOKBACK > -1: + first_allowed_key = ((q_abs // CHUNK_SIZE) - CHUNK_LOOKBACK) * CHUNK_SIZE + else: + first_allowed_key = q_abs - SLIDING_WINDOW + 1 last_allowed_key = context_len + qpos_hi # Convert to tile indices and clamp tile_start = tl.maximum(0, first_allowed_key // TILE_SIZE) @@ -370,9 +376,19 @@ def kernel_unified_attention_2d( query_abs_pos = context_len + query_pos[:, None] seq_mask = seq_offset[None, :] <= query_abs_pos - # Apply sliding window to base mask BEFORE mm_prefix OR. - # Order must match FlexAttention: (causal AND sliding_window) OR mm_prefix - if SLIDING_WINDOW > 0: + # Apply sliding window / chunked attention to base mask + # BEFORE mm_prefix OR. + # Order must match FlexAttention: + # (causal AND sliding_window) OR mm_prefix + if CHUNK_LOOKBACK > -1: + seq_mask = seq_mask & ( + ( + (context_len + query_pos[:, None]) // CHUNK_SIZE + - (seq_offset[None, :] // CHUNK_SIZE) + ) + <= CHUNK_LOOKBACK + ) + elif SLIDING_WINDOW > 0: seq_mask = seq_mask & ((query_abs_pos - seq_offset) < SLIDING_WINDOW) # PrefixLM: extend mask with bidirectional ranges for multimodal tokens. @@ -559,6 +575,8 @@ def kernel_unified_attention_3d( stride_vs_blk=0, stride_vs_slot=0, stride_vs_head=0, + CHUNK_LOOKBACK: tl.constexpr = -1, + CHUNK_SIZE: tl.constexpr = -1, ): q_block_global_idx = tl.program_id(0) kv_head_idx = tl.program_id(1) @@ -681,7 +699,11 @@ def kernel_unified_attention_3d( # where q_abs = context_len + q # The union of allowed key positions for this Q-block is: # [context_len + qpos_lo - SLIDING_WINDOW + 1, context_len + qpos_hi] - first_allowed_key = context_len + qpos_lo - SLIDING_WINDOW + 1 + q_abs = context_len + qpos_lo + if CHUNK_LOOKBACK > -1: + first_allowed_key = ((q_abs // CHUNK_SIZE) - CHUNK_LOOKBACK) * CHUNK_SIZE + else: + first_allowed_key = q_abs - SLIDING_WINDOW + 1 last_allowed_key = context_len + qpos_hi # Convert to tile indices and clamp tile_start = tl.maximum(0, first_allowed_key // TILE_SIZE) @@ -761,9 +783,19 @@ def kernel_unified_attention_3d( query_abs_pos = context_len + query_pos[:, None] seq_mask = seq_offset[None, :] <= query_abs_pos - # Apply sliding window to base mask BEFORE mm_prefix OR. - # Order must match FlexAttention: (causal AND sliding_window) OR mm_prefix - if SLIDING_WINDOW > 0: + # Apply sliding window / chunked attention to base mask + # BEFORE mm_prefix OR. + # Order must match FlexAttention: + # (causal AND sliding_window) OR mm_prefix + if CHUNK_LOOKBACK > -1: + seq_mask = seq_mask & ( + ( + (context_len + query_pos[:, None]) // CHUNK_SIZE + - (seq_offset[None, :] // CHUNK_SIZE) + ) + <= CHUNK_LOOKBACK + ) + elif SLIDING_WINDOW > 0: seq_mask = seq_mask & ((query_abs_pos - seq_offset) < SLIDING_WINDOW) # PrefixLM: extend mask with bidirectional ranges for multimodal tokens. @@ -1046,6 +1078,8 @@ def unified_attention( kv_quant_mode: KVQuantMode = KVQuantMode.NONE, k_scale_cache=None, # [num_blocks, block_size, num_kv_heads] float32 v_scale_cache=None, # [num_blocks, block_size, num_kv_heads] float32 + # Chunked attention: restrict attention to aligned blocks with lookback. + chunk_lookback=-1, ): assert causal, "Only causal attention is supported" assert q_descale is None, "Q scales not supported" @@ -1093,6 +1127,15 @@ def unified_attention( # Tile sizes for prefill and decode. Gemma3 models use optimized values. # Note: tile size must be at least 32 for fp8 (element_size == 1). sliding_window_val = 1 + window_size[0] if window_size[0] >= 0 else 0 + + # Compute chunked block size from sliding window if needed. + chunk_size = -1 + if sliding_window_val > 0 and chunk_lookback > -1: + chunk_size = sliding_window_val // (chunk_lookback + 1) + assert chunk_size > 0, "sliding_window must be > chunk_lookback+1" + elif sliding_window_val <= 0: + chunk_lookback = -1 + TILE_SIZE_PREFILL = _get_tile_size( head_size, sliding_window_val, @@ -1184,6 +1227,8 @@ def unified_attention( stride_vs_blk=v_scale_cache.stride(0) if v_scale_cache is not None else 0, stride_vs_slot=v_scale_cache.stride(1) if v_scale_cache is not None else 0, stride_vs_head=v_scale_cache.stride(2) if v_scale_cache is not None else 0, + CHUNK_LOOKBACK=chunk_lookback, + CHUNK_SIZE=chunk_size, ) else: kernel_unified_attention_3d[ @@ -1245,6 +1290,8 @@ def unified_attention( stride_vs_blk=v_scale_cache.stride(0) if v_scale_cache is not None else 0, stride_vs_slot=v_scale_cache.stride(1) if v_scale_cache is not None else 0, stride_vs_head=v_scale_cache.stride(2) if v_scale_cache is not None else 0, + CHUNK_LOOKBACK=chunk_lookback, + CHUNK_SIZE=chunk_size, ) reduce_segments[(q.shape[0], num_query_heads)]( output_ptr=out,