[Model] Pixtral: use packed multimodal encoder attention (#52185)

Signed-off-by: Oliver Holworthy <1216955+oliverholworthy@users.noreply.github.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Oliver Holworthy
2026-08-26 14:14:40 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent c47ca4aaeb
commit 28b484e5d7
2 changed files with 158 additions and 72 deletions
@@ -5,6 +5,7 @@ from dataclasses import asdict
from typing import TYPE_CHECKING, Any
import pytest
import torch
from mistral_common.multimodal import download_image
from mistral_common.protocol.instruct.chunk import ImageURLChunk
from mistral_common.protocol.instruct.request import ChatCompletionRequest
@@ -15,7 +16,9 @@ from transformers import AutoProcessor
from vllm import SamplingParams, TextPrompt, TokensPrompt
from vllm.inputs import MultiModalDataBuiltins
from vllm.logprobs import Logprob, SampleLogprobs
from vllm.model_executor.models.pixtral import _make_packed_sequence_metadata
from vllm.platforms import current_platform
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from ....utils import VLLM_PATH, large_gpu_test
from ...utils import check_logprobs_close
@@ -123,6 +126,37 @@ FIXTURE_LOGPROBS_CHAT = {
OutputsLogprobs = list[tuple[list[int], str, SampleLogprobs | None]]
@pytest.mark.parametrize(
"backend",
[
AttentionBackendEnum.FLASH_ATTN,
AttentionBackendEnum.FLASHINFER,
AttentionBackendEnum.TORCH_SDPA,
],
)
def test_packed_sequence_metadata(backend: AttentionBackendEnum) -> None:
cu_seqlens, max_seqlen, sequence_lengths = _make_packed_sequence_metadata(
[4, 6],
backend,
hidden_size=64,
tp_size=1,
device=torch.device("cpu"),
)
assert cu_seqlens.dtype == torch.int32
assert max_seqlen.dtype == torch.int32
if backend == AttentionBackendEnum.FLASHINFER:
assert max_seqlen.item() >= 6
assert sequence_lengths is not None
assert sequence_lengths.dtype == torch.int32
assert len(cu_seqlens) % 2 == 0
else:
expected_max_seqlen = 6 if backend == AttentionBackendEnum.FLASH_ATTN else 0
assert max_seqlen.item() == expected_max_seqlen
assert cu_seqlens.tolist() == [0, 4, 10]
assert sequence_lengths is None
# For the test author to store golden output in JSON
def _dump_outputs_w_logprobs(
outputs: OutputsLogprobs,
+124 -72
View File
@@ -6,6 +6,7 @@ from collections.abc import Iterable, Mapping, Sequence
from dataclasses import dataclass, fields
from typing import Annotated, Literal
import numpy as np
import torch
import torch.nn as nn
from mistral_common.protocol.instruct.chunk import ImageChunk, TextChunk
@@ -26,6 +27,7 @@ from vllm.config.multimodal import BaseDummyOptions
from vllm.distributed import divide, get_tensor_model_parallel_world_size
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import SiluAndMul, get_act_and_mul_fn
from vllm.model_executor.layers.attention import MMEncoderAttention
from vllm.model_executor.layers.conv import Conv2dLayer
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
@@ -59,7 +61,6 @@ from vllm.multimodal.processing.processor import (
PromptUpdate,
PromptUpdateDetails,
)
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.tokenizers import cached_tokenizer_from_config
from vllm.tokenizers.mistral import MistralTokenizer
@@ -69,6 +70,7 @@ from vllm.transformers_utils.processors.pixtral import (
)
from vllm.utils.collection_utils import is_list_of
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from .interfaces import (
MultiModalEmbeddings,
@@ -87,21 +89,37 @@ from .vision import (
resolve_visual_encoder_outputs,
)
try:
# Note: vLLM does not install xformers by default.
from xformers import ops as xops
if current_platform.is_cuda() and current_platform.has_device_capability(100):
# Xformers FA is not compatible with B200
USE_XFORMERS_OPS = False
else:
USE_XFORMERS_OPS = True
except ImportError:
USE_XFORMERS_OPS = False
PATCH_MERGE = "patch_merge"
def _make_packed_sequence_metadata(
sequence_lengths: list[int],
attn_backend: AttentionBackendEnum,
hidden_size: int,
tp_size: int,
device: torch.device,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
lengths = np.array(sequence_lengths, dtype=np.int32)
cu_seqlens = np.concatenate(
[np.zeros(1, dtype=np.int32), lengths.cumsum(dtype=np.int32)]
)
sequence_lengths_tensor = MMEncoderAttention.maybe_compute_seq_lens(
attn_backend, cu_seqlens, device
)
max_seqlen = torch.tensor(
MMEncoderAttention.compute_max_seqlen(attn_backend, cu_seqlens),
dtype=torch.int32,
)
cu_seqlens_tensor = MMEncoderAttention.maybe_recompute_cu_seqlens(
attn_backend,
cu_seqlens,
hidden_size,
tp_size,
device,
)
return cu_seqlens_tensor, max_seqlen, sequence_lengths_tensor
def _is_layer_none_or_staged(layer: nn.Module) -> bool:
return layer is None or isinstance(layer, StageMissingLayer)
@@ -766,12 +784,19 @@ class Attention(nn.Module):
tp_size = 1 if disable_tp else get_tensor_model_parallel_world_size()
self.n_heads = divide(args.num_attention_heads, tp_size)
self.attn = MMEncoderAttention(
num_heads=self.n_heads,
head_size=self.head_dim,
prefix=f"{prefix}.attn",
)
def forward(
self,
x: torch.Tensor,
mask: torch.Tensor,
freqs_cis: torch.Tensor,
cu_seqlens: torch.Tensor,
max_seqlen: torch.Tensor,
sequence_lengths: torch.Tensor | None,
) -> torch.Tensor:
batch, patches, _ = x.shape
@@ -782,15 +807,14 @@ class Attention(nn.Module):
v = v.reshape(batch, patches, self.n_heads, self.head_dim)
q, k = apply_rotary_emb_vit(q, k, freqs_cis=freqs_cis)
if USE_XFORMERS_OPS:
out = xops.memory_efficient_attention(q, k, v, attn_bias=mask)
else:
q = q.transpose(1, 2)
k = k.transpose(1, 2)
v = v.transpose(1, 2)
out = nn.functional.scaled_dot_product_attention(q, k, v, attn_mask=mask)
out = out.transpose(1, 2)
out = self.attn(
q,
k,
v,
cu_seqlens=cu_seqlens,
max_seqlen=max_seqlen,
sequence_lengths=sequence_lengths,
)
out = out.reshape(batch, patches, self.n_heads * self.head_dim)
out, _ = self.o_proj(out)
@@ -825,11 +849,17 @@ class TransformerBlock(nn.Module):
def forward(
self,
x: torch.Tensor,
mask: torch.Tensor,
freqs_cis: torch.Tensor,
cu_seqlens: torch.Tensor,
max_seqlen: torch.Tensor,
sequence_lengths: torch.Tensor | None,
) -> torch.Tensor:
r = self.attention.forward(
self.attention_norm(x), mask=mask, freqs_cis=freqs_cis
self.attention_norm(x),
freqs_cis=freqs_cis,
cu_seqlens=cu_seqlens,
max_seqlen=max_seqlen,
sequence_lengths=sequence_lengths,
)
h = x + r
r = self.feed_forward.forward(self.ffn_norm(h))
@@ -860,11 +890,19 @@ class Transformer(nn.Module):
def forward(
self,
x: torch.Tensor,
mask: torch.Tensor,
freqs_cis: torch.Tensor | None,
cu_seqlens: torch.Tensor,
max_seqlen: torch.Tensor,
sequence_lengths: torch.Tensor | None,
) -> torch.Tensor:
for layer in self.layers:
x = layer(x, mask=mask, freqs_cis=freqs_cis)
x = layer(
x,
freqs_cis=freqs_cis,
cu_seqlens=cu_seqlens,
max_seqlen=max_seqlen,
sequence_lengths=sequence_lengths,
)
return x
@@ -971,20 +1009,21 @@ class VisionTransformer(nn.Module):
positions = position_meshgrid(patch_embeds_list).to(self.device)
freqs_cis = self.freqs_cis[positions[:, 0], positions[:, 1]]
# pass through Transformer with a block diagonal mask delimiting images
if USE_XFORMERS_OPS:
mask = xops.fmha.attn_bias.BlockDiagonalMask.from_seqlens(
[p.shape[-2] * p.shape[-1] for p in patch_embeds_list],
)
else:
from transformers.models.pixtral.modeling_pixtral import (
generate_block_attention_mask,
)
mask = generate_block_attention_mask(
[p.shape[-2] * p.shape[-1] for p in patch_embeds_list], patch_embeds
)
out = self.transformer(patch_embeds, mask=mask, freqs_cis=freqs_cis)
attention = self.transformer.layers[0].attention.attn
cu_seqlens, max_seqlen, sequence_lengths = _make_packed_sequence_metadata(
embed_sizes,
attention.attn_backend,
self.args.hidden_size,
1 if is_vit_use_data_parallel() else get_tensor_model_parallel_world_size(),
patch_embeds.device,
)
out = self.transformer(
patch_embeds,
freqs_cis=freqs_cis,
cu_seqlens=cu_seqlens,
max_seqlen=max_seqlen,
sequence_lengths=sequence_lengths,
)
# squeeze dim 0 and split into separate tensors for each image
return torch.split(out.squeeze(0), embed_sizes)
@@ -1244,36 +1283,39 @@ class PixtralHFAttention(nn.Module):
1 if use_data_parallel else get_tensor_model_parallel_world_size()
)
self.n_heads = divide(config.num_attention_heads, self.tp_size)
self.attn = MMEncoderAttention(
num_heads=self.n_heads,
head_size=self.head_dim,
prefix=f"{prefix}.attn",
)
def forward(
self,
hidden_states: torch.Tensor,
attention_mask: torch.Tensor,
position_embeddings: torch.Tensor,
cu_seqlens: torch.Tensor,
max_seqlen: torch.Tensor,
sequence_lengths: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
batch, patches, _ = hidden_states.size()
qkv_states, _ = self.qkv_proj(hidden_states)
q, k, v = qkv_states.chunk(3, dim=-1)
# Transpose q and k to apply HF's Rotary Position Embedding
# Transpose q and k to apply HF's Rotary Position Embedding.
q = q.view(batch, patches, self.n_heads, self.head_dim).transpose(1, 2)
k = k.view(batch, patches, self.n_heads, self.head_dim).transpose(1, 2)
v = v.view(batch, patches, self.n_heads, self.head_dim)
cos, sin = position_embeddings
q, k = apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=0)
if USE_XFORMERS_OPS:
# Transpose q and k back for attention
q = q.transpose(1, 2).contiguous()
k = k.transpose(1, 2).contiguous()
out = xops.memory_efficient_attention(q, k, v, attn_bias=attention_mask)
else:
v = v.transpose(1, 2)
out = nn.functional.scaled_dot_product_attention(
q, k, v, attn_mask=attention_mask
)
out = out.transpose(1, 2)
out = self.attn(
q.transpose(1, 2).contiguous(),
k.transpose(1, 2).contiguous(),
v,
cu_seqlens=cu_seqlens,
max_seqlen=max_seqlen,
sequence_lengths=sequence_lengths,
)
out = out.reshape(batch, patches, self.n_heads * self.head_dim)
attn_output, _ = self.o_proj(out)
@@ -1307,13 +1349,17 @@ class PixtralHFTransformerBlock(nn.Module):
def forward(
self,
hidden_states: torch.Tensor,
attention_mask: torch.Tensor,
position_embeddings: torch.Tensor,
cu_seqlens: torch.Tensor,
max_seqlen: torch.Tensor,
sequence_lengths: torch.Tensor | None,
) -> torch.Tensor:
r, _ = self.attention.forward(
self.attention_norm(hidden_states),
attention_mask=attention_mask,
position_embeddings=position_embeddings,
cu_seqlens=cu_seqlens,
max_seqlen=max_seqlen,
sequence_lengths=sequence_lengths,
)
h = hidden_states + r
r = self.feed_forward.forward(self.ffn_norm(h))
@@ -1351,14 +1397,22 @@ class PixtralHFTransformer(nn.Module):
def forward(
self,
x: torch.Tensor,
attention_mask: torch.Tensor,
position_embeddings: torch.Tensor,
cu_seqlens: torch.Tensor,
max_seqlen: torch.Tensor,
sequence_lengths: torch.Tensor | None,
return_all_hidden_states: bool,
) -> torch.Tensor:
hidden_states_pool = [x]
for layer in self.layers:
x = layer(x, attention_mask, position_embeddings)
x = layer(
x,
position_embeddings,
cu_seqlens,
max_seqlen,
sequence_lengths,
)
if return_all_hidden_states:
hidden_states_pool.append(x)
# If we have multiple feature sample layers, we return all hidden
@@ -1453,23 +1507,21 @@ class PixtralHFVisionModel(nn.Module):
).to(self.device)
position_embedding = self.patch_positional_embedding(patch_embeds, position_ids)
if USE_XFORMERS_OPS:
attention_mask = xops.fmha.attn_bias.BlockDiagonalMask.from_seqlens(
[p.shape[-2] * p.shape[-1] for p in patch_embeds_list],
)
else:
from transformers.models.pixtral.modeling_pixtral import (
generate_block_attention_mask,
)
attention_mask = generate_block_attention_mask(
[p.shape[-2] * p.shape[-1] for p in patch_embeds_list], patch_embeds
)
attention = self.transformer.layers[0].attention.attn
cu_seqlens, max_seqlen, sequence_lengths = _make_packed_sequence_metadata(
embed_sizes,
attention.attn_backend,
self.config.hidden_size,
self.transformer.layers[0].attention.tp_size,
patch_embeds.device,
)
out = self.transformer(
patch_embeds,
attention_mask,
position_embedding,
cu_seqlens,
max_seqlen,
sequence_lengths,
return_all_hidden_states=select_layers is not None,
)