From 28b484e5d741519feb72e5c0bf6ebc3b08176292 Mon Sep 17 00:00:00 2001 From: Oliver Holworthy <1216955+oliverholworthy@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:14:40 +0100 Subject: [PATCH] [Model] Pixtral: use packed multimodal encoder attention (#52185) Signed-off-by: Oliver Holworthy <1216955+oliverholworthy@users.noreply.github.com> Co-authored-by: OpenAI Codex --- .../multimodal/generation/test_pixtral.py | 34 +++ vllm/model_executor/models/pixtral.py | 196 +++++++++++------- 2 files changed, 158 insertions(+), 72 deletions(-) diff --git a/tests/models/multimodal/generation/test_pixtral.py b/tests/models/multimodal/generation/test_pixtral.py index 2ce732342b..9be8f8efab 100644 --- a/tests/models/multimodal/generation/test_pixtral.py +++ b/tests/models/multimodal/generation/test_pixtral.py @@ -5,6 +5,7 @@ from dataclasses import asdict from typing import TYPE_CHECKING, Any import pytest +import torch from mistral_common.multimodal import download_image from mistral_common.protocol.instruct.chunk import ImageURLChunk from mistral_common.protocol.instruct.request import ChatCompletionRequest @@ -15,7 +16,9 @@ from transformers import AutoProcessor from vllm import SamplingParams, TextPrompt, TokensPrompt from vllm.inputs import MultiModalDataBuiltins from vllm.logprobs import Logprob, SampleLogprobs +from vllm.model_executor.models.pixtral import _make_packed_sequence_metadata from vllm.platforms import current_platform +from vllm.v1.attention.backends.registry import AttentionBackendEnum from ....utils import VLLM_PATH, large_gpu_test from ...utils import check_logprobs_close @@ -123,6 +126,37 @@ FIXTURE_LOGPROBS_CHAT = { OutputsLogprobs = list[tuple[list[int], str, SampleLogprobs | None]] +@pytest.mark.parametrize( + "backend", + [ + AttentionBackendEnum.FLASH_ATTN, + AttentionBackendEnum.FLASHINFER, + AttentionBackendEnum.TORCH_SDPA, + ], +) +def test_packed_sequence_metadata(backend: AttentionBackendEnum) -> None: + cu_seqlens, max_seqlen, sequence_lengths = _make_packed_sequence_metadata( + [4, 6], + backend, + hidden_size=64, + tp_size=1, + device=torch.device("cpu"), + ) + + assert cu_seqlens.dtype == torch.int32 + assert max_seqlen.dtype == torch.int32 + if backend == AttentionBackendEnum.FLASHINFER: + assert max_seqlen.item() >= 6 + assert sequence_lengths is not None + assert sequence_lengths.dtype == torch.int32 + assert len(cu_seqlens) % 2 == 0 + else: + expected_max_seqlen = 6 if backend == AttentionBackendEnum.FLASH_ATTN else 0 + assert max_seqlen.item() == expected_max_seqlen + assert cu_seqlens.tolist() == [0, 4, 10] + assert sequence_lengths is None + + # For the test author to store golden output in JSON def _dump_outputs_w_logprobs( outputs: OutputsLogprobs, diff --git a/vllm/model_executor/models/pixtral.py b/vllm/model_executor/models/pixtral.py index adfba2e188..b6848b09de 100644 --- a/vllm/model_executor/models/pixtral.py +++ b/vllm/model_executor/models/pixtral.py @@ -6,6 +6,7 @@ from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass, fields from typing import Annotated, Literal +import numpy as np import torch import torch.nn as nn from mistral_common.protocol.instruct.chunk import ImageChunk, TextChunk @@ -26,6 +27,7 @@ from vllm.config.multimodal import BaseDummyOptions from vllm.distributed import divide, get_tensor_model_parallel_world_size from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.activation import SiluAndMul, get_act_and_mul_fn +from vllm.model_executor.layers.attention import MMEncoderAttention from vllm.model_executor.layers.conv import Conv2dLayer from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( @@ -59,7 +61,6 @@ from vllm.multimodal.processing.processor import ( PromptUpdate, PromptUpdateDetails, ) -from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors from vllm.tokenizers import cached_tokenizer_from_config from vllm.tokenizers.mistral import MistralTokenizer @@ -69,6 +70,7 @@ from vllm.transformers_utils.processors.pixtral import ( ) from vllm.utils.collection_utils import is_list_of from vllm.utils.tensor_schema import TensorSchema, TensorShape +from vllm.v1.attention.backends.registry import AttentionBackendEnum from .interfaces import ( MultiModalEmbeddings, @@ -87,21 +89,37 @@ from .vision import ( resolve_visual_encoder_outputs, ) -try: - # Note: vLLM does not install xformers by default. - from xformers import ops as xops - - if current_platform.is_cuda() and current_platform.has_device_capability(100): - # Xformers FA is not compatible with B200 - USE_XFORMERS_OPS = False - else: - USE_XFORMERS_OPS = True -except ImportError: - USE_XFORMERS_OPS = False - PATCH_MERGE = "patch_merge" +def _make_packed_sequence_metadata( + sequence_lengths: list[int], + attn_backend: AttentionBackendEnum, + hidden_size: int, + tp_size: int, + device: torch.device, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]: + lengths = np.array(sequence_lengths, dtype=np.int32) + cu_seqlens = np.concatenate( + [np.zeros(1, dtype=np.int32), lengths.cumsum(dtype=np.int32)] + ) + sequence_lengths_tensor = MMEncoderAttention.maybe_compute_seq_lens( + attn_backend, cu_seqlens, device + ) + max_seqlen = torch.tensor( + MMEncoderAttention.compute_max_seqlen(attn_backend, cu_seqlens), + dtype=torch.int32, + ) + cu_seqlens_tensor = MMEncoderAttention.maybe_recompute_cu_seqlens( + attn_backend, + cu_seqlens, + hidden_size, + tp_size, + device, + ) + return cu_seqlens_tensor, max_seqlen, sequence_lengths_tensor + + def _is_layer_none_or_staged(layer: nn.Module) -> bool: return layer is None or isinstance(layer, StageMissingLayer) @@ -766,12 +784,19 @@ class Attention(nn.Module): tp_size = 1 if disable_tp else get_tensor_model_parallel_world_size() self.n_heads = divide(args.num_attention_heads, tp_size) + self.attn = MMEncoderAttention( + num_heads=self.n_heads, + head_size=self.head_dim, + prefix=f"{prefix}.attn", + ) def forward( self, x: torch.Tensor, - mask: torch.Tensor, freqs_cis: torch.Tensor, + cu_seqlens: torch.Tensor, + max_seqlen: torch.Tensor, + sequence_lengths: torch.Tensor | None, ) -> torch.Tensor: batch, patches, _ = x.shape @@ -782,15 +807,14 @@ class Attention(nn.Module): v = v.reshape(batch, patches, self.n_heads, self.head_dim) q, k = apply_rotary_emb_vit(q, k, freqs_cis=freqs_cis) - - if USE_XFORMERS_OPS: - out = xops.memory_efficient_attention(q, k, v, attn_bias=mask) - else: - q = q.transpose(1, 2) - k = k.transpose(1, 2) - v = v.transpose(1, 2) - out = nn.functional.scaled_dot_product_attention(q, k, v, attn_mask=mask) - out = out.transpose(1, 2) + out = self.attn( + q, + k, + v, + cu_seqlens=cu_seqlens, + max_seqlen=max_seqlen, + sequence_lengths=sequence_lengths, + ) out = out.reshape(batch, patches, self.n_heads * self.head_dim) out, _ = self.o_proj(out) @@ -825,11 +849,17 @@ class TransformerBlock(nn.Module): def forward( self, x: torch.Tensor, - mask: torch.Tensor, freqs_cis: torch.Tensor, + cu_seqlens: torch.Tensor, + max_seqlen: torch.Tensor, + sequence_lengths: torch.Tensor | None, ) -> torch.Tensor: r = self.attention.forward( - self.attention_norm(x), mask=mask, freqs_cis=freqs_cis + self.attention_norm(x), + freqs_cis=freqs_cis, + cu_seqlens=cu_seqlens, + max_seqlen=max_seqlen, + sequence_lengths=sequence_lengths, ) h = x + r r = self.feed_forward.forward(self.ffn_norm(h)) @@ -860,11 +890,19 @@ class Transformer(nn.Module): def forward( self, x: torch.Tensor, - mask: torch.Tensor, freqs_cis: torch.Tensor | None, + cu_seqlens: torch.Tensor, + max_seqlen: torch.Tensor, + sequence_lengths: torch.Tensor | None, ) -> torch.Tensor: for layer in self.layers: - x = layer(x, mask=mask, freqs_cis=freqs_cis) + x = layer( + x, + freqs_cis=freqs_cis, + cu_seqlens=cu_seqlens, + max_seqlen=max_seqlen, + sequence_lengths=sequence_lengths, + ) return x @@ -971,20 +1009,21 @@ class VisionTransformer(nn.Module): positions = position_meshgrid(patch_embeds_list).to(self.device) freqs_cis = self.freqs_cis[positions[:, 0], positions[:, 1]] - # pass through Transformer with a block diagonal mask delimiting images - if USE_XFORMERS_OPS: - mask = xops.fmha.attn_bias.BlockDiagonalMask.from_seqlens( - [p.shape[-2] * p.shape[-1] for p in patch_embeds_list], - ) - else: - from transformers.models.pixtral.modeling_pixtral import ( - generate_block_attention_mask, - ) - - mask = generate_block_attention_mask( - [p.shape[-2] * p.shape[-1] for p in patch_embeds_list], patch_embeds - ) - out = self.transformer(patch_embeds, mask=mask, freqs_cis=freqs_cis) + attention = self.transformer.layers[0].attention.attn + cu_seqlens, max_seqlen, sequence_lengths = _make_packed_sequence_metadata( + embed_sizes, + attention.attn_backend, + self.args.hidden_size, + 1 if is_vit_use_data_parallel() else get_tensor_model_parallel_world_size(), + patch_embeds.device, + ) + out = self.transformer( + patch_embeds, + freqs_cis=freqs_cis, + cu_seqlens=cu_seqlens, + max_seqlen=max_seqlen, + sequence_lengths=sequence_lengths, + ) # squeeze dim 0 and split into separate tensors for each image return torch.split(out.squeeze(0), embed_sizes) @@ -1244,36 +1283,39 @@ class PixtralHFAttention(nn.Module): 1 if use_data_parallel else get_tensor_model_parallel_world_size() ) self.n_heads = divide(config.num_attention_heads, self.tp_size) + self.attn = MMEncoderAttention( + num_heads=self.n_heads, + head_size=self.head_dim, + prefix=f"{prefix}.attn", + ) def forward( self, hidden_states: torch.Tensor, - attention_mask: torch.Tensor, position_embeddings: torch.Tensor, + cu_seqlens: torch.Tensor, + max_seqlen: torch.Tensor, + sequence_lengths: torch.Tensor | None, ) -> tuple[torch.Tensor, torch.Tensor | None]: batch, patches, _ = hidden_states.size() qkv_states, _ = self.qkv_proj(hidden_states) q, k, v = qkv_states.chunk(3, dim=-1) - # Transpose q and k to apply HF's Rotary Position Embedding + # Transpose q and k to apply HF's Rotary Position Embedding. q = q.view(batch, patches, self.n_heads, self.head_dim).transpose(1, 2) k = k.view(batch, patches, self.n_heads, self.head_dim).transpose(1, 2) v = v.view(batch, patches, self.n_heads, self.head_dim) cos, sin = position_embeddings q, k = apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=0) - - if USE_XFORMERS_OPS: - # Transpose q and k back for attention - q = q.transpose(1, 2).contiguous() - k = k.transpose(1, 2).contiguous() - out = xops.memory_efficient_attention(q, k, v, attn_bias=attention_mask) - else: - v = v.transpose(1, 2) - out = nn.functional.scaled_dot_product_attention( - q, k, v, attn_mask=attention_mask - ) - out = out.transpose(1, 2) + out = self.attn( + q.transpose(1, 2).contiguous(), + k.transpose(1, 2).contiguous(), + v, + cu_seqlens=cu_seqlens, + max_seqlen=max_seqlen, + sequence_lengths=sequence_lengths, + ) out = out.reshape(batch, patches, self.n_heads * self.head_dim) attn_output, _ = self.o_proj(out) @@ -1307,13 +1349,17 @@ class PixtralHFTransformerBlock(nn.Module): def forward( self, hidden_states: torch.Tensor, - attention_mask: torch.Tensor, position_embeddings: torch.Tensor, + cu_seqlens: torch.Tensor, + max_seqlen: torch.Tensor, + sequence_lengths: torch.Tensor | None, ) -> torch.Tensor: r, _ = self.attention.forward( self.attention_norm(hidden_states), - attention_mask=attention_mask, position_embeddings=position_embeddings, + cu_seqlens=cu_seqlens, + max_seqlen=max_seqlen, + sequence_lengths=sequence_lengths, ) h = hidden_states + r r = self.feed_forward.forward(self.ffn_norm(h)) @@ -1351,14 +1397,22 @@ class PixtralHFTransformer(nn.Module): def forward( self, x: torch.Tensor, - attention_mask: torch.Tensor, position_embeddings: torch.Tensor, + cu_seqlens: torch.Tensor, + max_seqlen: torch.Tensor, + sequence_lengths: torch.Tensor | None, return_all_hidden_states: bool, ) -> torch.Tensor: hidden_states_pool = [x] for layer in self.layers: - x = layer(x, attention_mask, position_embeddings) + x = layer( + x, + position_embeddings, + cu_seqlens, + max_seqlen, + sequence_lengths, + ) if return_all_hidden_states: hidden_states_pool.append(x) # If we have multiple feature sample layers, we return all hidden @@ -1453,23 +1507,21 @@ class PixtralHFVisionModel(nn.Module): ).to(self.device) position_embedding = self.patch_positional_embedding(patch_embeds, position_ids) - if USE_XFORMERS_OPS: - attention_mask = xops.fmha.attn_bias.BlockDiagonalMask.from_seqlens( - [p.shape[-2] * p.shape[-1] for p in patch_embeds_list], - ) - else: - from transformers.models.pixtral.modeling_pixtral import ( - generate_block_attention_mask, - ) - - attention_mask = generate_block_attention_mask( - [p.shape[-2] * p.shape[-1] for p in patch_embeds_list], patch_embeds - ) + attention = self.transformer.layers[0].attention.attn + cu_seqlens, max_seqlen, sequence_lengths = _make_packed_sequence_metadata( + embed_sizes, + attention.attn_backend, + self.config.hidden_size, + self.transformer.layers[0].attention.tp_size, + patch_embeds.device, + ) out = self.transformer( patch_embeds, - attention_mask, position_embedding, + cu_seqlens, + max_seqlen, + sequence_lengths, return_all_hidden_states=select_layers is not None, )