[Model] Pixtral: use packed multimodal encoder attention (#52185)
Signed-off-by: Oliver Holworthy <1216955+oliverholworthy@users.noreply.github.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
c47ca4aaeb
commit
28b484e5d7
@@ -5,6 +5,7 @@ from dataclasses import asdict
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from mistral_common.multimodal import download_image
|
||||
from mistral_common.protocol.instruct.chunk import ImageURLChunk
|
||||
from mistral_common.protocol.instruct.request import ChatCompletionRequest
|
||||
@@ -15,7 +16,9 @@ from transformers import AutoProcessor
|
||||
from vllm import SamplingParams, TextPrompt, TokensPrompt
|
||||
from vllm.inputs import MultiModalDataBuiltins
|
||||
from vllm.logprobs import Logprob, SampleLogprobs
|
||||
from vllm.model_executor.models.pixtral import _make_packed_sequence_metadata
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
from ....utils import VLLM_PATH, large_gpu_test
|
||||
from ...utils import check_logprobs_close
|
||||
@@ -123,6 +126,37 @@ FIXTURE_LOGPROBS_CHAT = {
|
||||
OutputsLogprobs = list[tuple[list[int], str, SampleLogprobs | None]]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"backend",
|
||||
[
|
||||
AttentionBackendEnum.FLASH_ATTN,
|
||||
AttentionBackendEnum.FLASHINFER,
|
||||
AttentionBackendEnum.TORCH_SDPA,
|
||||
],
|
||||
)
|
||||
def test_packed_sequence_metadata(backend: AttentionBackendEnum) -> None:
|
||||
cu_seqlens, max_seqlen, sequence_lengths = _make_packed_sequence_metadata(
|
||||
[4, 6],
|
||||
backend,
|
||||
hidden_size=64,
|
||||
tp_size=1,
|
||||
device=torch.device("cpu"),
|
||||
)
|
||||
|
||||
assert cu_seqlens.dtype == torch.int32
|
||||
assert max_seqlen.dtype == torch.int32
|
||||
if backend == AttentionBackendEnum.FLASHINFER:
|
||||
assert max_seqlen.item() >= 6
|
||||
assert sequence_lengths is not None
|
||||
assert sequence_lengths.dtype == torch.int32
|
||||
assert len(cu_seqlens) % 2 == 0
|
||||
else:
|
||||
expected_max_seqlen = 6 if backend == AttentionBackendEnum.FLASH_ATTN else 0
|
||||
assert max_seqlen.item() == expected_max_seqlen
|
||||
assert cu_seqlens.tolist() == [0, 4, 10]
|
||||
assert sequence_lengths is None
|
||||
|
||||
|
||||
# For the test author to store golden output in JSON
|
||||
def _dump_outputs_w_logprobs(
|
||||
outputs: OutputsLogprobs,
|
||||
|
||||
@@ -6,6 +6,7 @@ from collections.abc import Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass, fields
|
||||
from typing import Annotated, Literal
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from mistral_common.protocol.instruct.chunk import ImageChunk, TextChunk
|
||||
@@ -26,6 +27,7 @@ from vllm.config.multimodal import BaseDummyOptions
|
||||
from vllm.distributed import divide, get_tensor_model_parallel_world_size
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.model_executor.layers.activation import SiluAndMul, get_act_and_mul_fn
|
||||
from vllm.model_executor.layers.attention import MMEncoderAttention
|
||||
from vllm.model_executor.layers.conv import Conv2dLayer
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
@@ -59,7 +61,6 @@ from vllm.multimodal.processing.processor import (
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers import cached_tokenizer_from_config
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
@@ -69,6 +70,7 @@ from vllm.transformers_utils.processors.pixtral import (
|
||||
)
|
||||
from vllm.utils.collection_utils import is_list_of
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
from .interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
@@ -87,21 +89,37 @@ from .vision import (
|
||||
resolve_visual_encoder_outputs,
|
||||
)
|
||||
|
||||
try:
|
||||
# Note: vLLM does not install xformers by default.
|
||||
from xformers import ops as xops
|
||||
|
||||
if current_platform.is_cuda() and current_platform.has_device_capability(100):
|
||||
# Xformers FA is not compatible with B200
|
||||
USE_XFORMERS_OPS = False
|
||||
else:
|
||||
USE_XFORMERS_OPS = True
|
||||
except ImportError:
|
||||
USE_XFORMERS_OPS = False
|
||||
|
||||
PATCH_MERGE = "patch_merge"
|
||||
|
||||
|
||||
def _make_packed_sequence_metadata(
|
||||
sequence_lengths: list[int],
|
||||
attn_backend: AttentionBackendEnum,
|
||||
hidden_size: int,
|
||||
tp_size: int,
|
||||
device: torch.device,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
|
||||
lengths = np.array(sequence_lengths, dtype=np.int32)
|
||||
cu_seqlens = np.concatenate(
|
||||
[np.zeros(1, dtype=np.int32), lengths.cumsum(dtype=np.int32)]
|
||||
)
|
||||
sequence_lengths_tensor = MMEncoderAttention.maybe_compute_seq_lens(
|
||||
attn_backend, cu_seqlens, device
|
||||
)
|
||||
max_seqlen = torch.tensor(
|
||||
MMEncoderAttention.compute_max_seqlen(attn_backend, cu_seqlens),
|
||||
dtype=torch.int32,
|
||||
)
|
||||
cu_seqlens_tensor = MMEncoderAttention.maybe_recompute_cu_seqlens(
|
||||
attn_backend,
|
||||
cu_seqlens,
|
||||
hidden_size,
|
||||
tp_size,
|
||||
device,
|
||||
)
|
||||
return cu_seqlens_tensor, max_seqlen, sequence_lengths_tensor
|
||||
|
||||
|
||||
def _is_layer_none_or_staged(layer: nn.Module) -> bool:
|
||||
return layer is None or isinstance(layer, StageMissingLayer)
|
||||
|
||||
@@ -766,12 +784,19 @@ class Attention(nn.Module):
|
||||
|
||||
tp_size = 1 if disable_tp else get_tensor_model_parallel_world_size()
|
||||
self.n_heads = divide(args.num_attention_heads, tp_size)
|
||||
self.attn = MMEncoderAttention(
|
||||
num_heads=self.n_heads,
|
||||
head_size=self.head_dim,
|
||||
prefix=f"{prefix}.attn",
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
mask: torch.Tensor,
|
||||
freqs_cis: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
max_seqlen: torch.Tensor,
|
||||
sequence_lengths: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
batch, patches, _ = x.shape
|
||||
|
||||
@@ -782,15 +807,14 @@ class Attention(nn.Module):
|
||||
v = v.reshape(batch, patches, self.n_heads, self.head_dim)
|
||||
|
||||
q, k = apply_rotary_emb_vit(q, k, freqs_cis=freqs_cis)
|
||||
|
||||
if USE_XFORMERS_OPS:
|
||||
out = xops.memory_efficient_attention(q, k, v, attn_bias=mask)
|
||||
else:
|
||||
q = q.transpose(1, 2)
|
||||
k = k.transpose(1, 2)
|
||||
v = v.transpose(1, 2)
|
||||
out = nn.functional.scaled_dot_product_attention(q, k, v, attn_mask=mask)
|
||||
out = out.transpose(1, 2)
|
||||
out = self.attn(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
cu_seqlens=cu_seqlens,
|
||||
max_seqlen=max_seqlen,
|
||||
sequence_lengths=sequence_lengths,
|
||||
)
|
||||
|
||||
out = out.reshape(batch, patches, self.n_heads * self.head_dim)
|
||||
out, _ = self.o_proj(out)
|
||||
@@ -825,11 +849,17 @@ class TransformerBlock(nn.Module):
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
mask: torch.Tensor,
|
||||
freqs_cis: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
max_seqlen: torch.Tensor,
|
||||
sequence_lengths: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
r = self.attention.forward(
|
||||
self.attention_norm(x), mask=mask, freqs_cis=freqs_cis
|
||||
self.attention_norm(x),
|
||||
freqs_cis=freqs_cis,
|
||||
cu_seqlens=cu_seqlens,
|
||||
max_seqlen=max_seqlen,
|
||||
sequence_lengths=sequence_lengths,
|
||||
)
|
||||
h = x + r
|
||||
r = self.feed_forward.forward(self.ffn_norm(h))
|
||||
@@ -860,11 +890,19 @@ class Transformer(nn.Module):
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
mask: torch.Tensor,
|
||||
freqs_cis: torch.Tensor | None,
|
||||
cu_seqlens: torch.Tensor,
|
||||
max_seqlen: torch.Tensor,
|
||||
sequence_lengths: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
for layer in self.layers:
|
||||
x = layer(x, mask=mask, freqs_cis=freqs_cis)
|
||||
x = layer(
|
||||
x,
|
||||
freqs_cis=freqs_cis,
|
||||
cu_seqlens=cu_seqlens,
|
||||
max_seqlen=max_seqlen,
|
||||
sequence_lengths=sequence_lengths,
|
||||
)
|
||||
return x
|
||||
|
||||
|
||||
@@ -971,20 +1009,21 @@ class VisionTransformer(nn.Module):
|
||||
positions = position_meshgrid(patch_embeds_list).to(self.device)
|
||||
freqs_cis = self.freqs_cis[positions[:, 0], positions[:, 1]]
|
||||
|
||||
# pass through Transformer with a block diagonal mask delimiting images
|
||||
if USE_XFORMERS_OPS:
|
||||
mask = xops.fmha.attn_bias.BlockDiagonalMask.from_seqlens(
|
||||
[p.shape[-2] * p.shape[-1] for p in patch_embeds_list],
|
||||
)
|
||||
else:
|
||||
from transformers.models.pixtral.modeling_pixtral import (
|
||||
generate_block_attention_mask,
|
||||
)
|
||||
|
||||
mask = generate_block_attention_mask(
|
||||
[p.shape[-2] * p.shape[-1] for p in patch_embeds_list], patch_embeds
|
||||
)
|
||||
out = self.transformer(patch_embeds, mask=mask, freqs_cis=freqs_cis)
|
||||
attention = self.transformer.layers[0].attention.attn
|
||||
cu_seqlens, max_seqlen, sequence_lengths = _make_packed_sequence_metadata(
|
||||
embed_sizes,
|
||||
attention.attn_backend,
|
||||
self.args.hidden_size,
|
||||
1 if is_vit_use_data_parallel() else get_tensor_model_parallel_world_size(),
|
||||
patch_embeds.device,
|
||||
)
|
||||
out = self.transformer(
|
||||
patch_embeds,
|
||||
freqs_cis=freqs_cis,
|
||||
cu_seqlens=cu_seqlens,
|
||||
max_seqlen=max_seqlen,
|
||||
sequence_lengths=sequence_lengths,
|
||||
)
|
||||
|
||||
# squeeze dim 0 and split into separate tensors for each image
|
||||
return torch.split(out.squeeze(0), embed_sizes)
|
||||
@@ -1244,36 +1283,39 @@ class PixtralHFAttention(nn.Module):
|
||||
1 if use_data_parallel else get_tensor_model_parallel_world_size()
|
||||
)
|
||||
self.n_heads = divide(config.num_attention_heads, self.tp_size)
|
||||
self.attn = MMEncoderAttention(
|
||||
num_heads=self.n_heads,
|
||||
head_size=self.head_dim,
|
||||
prefix=f"{prefix}.attn",
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
attention_mask: torch.Tensor,
|
||||
position_embeddings: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
max_seqlen: torch.Tensor,
|
||||
sequence_lengths: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
batch, patches, _ = hidden_states.size()
|
||||
|
||||
qkv_states, _ = self.qkv_proj(hidden_states)
|
||||
q, k, v = qkv_states.chunk(3, dim=-1)
|
||||
|
||||
# Transpose q and k to apply HF's Rotary Position Embedding
|
||||
# Transpose q and k to apply HF's Rotary Position Embedding.
|
||||
q = q.view(batch, patches, self.n_heads, self.head_dim).transpose(1, 2)
|
||||
k = k.view(batch, patches, self.n_heads, self.head_dim).transpose(1, 2)
|
||||
v = v.view(batch, patches, self.n_heads, self.head_dim)
|
||||
cos, sin = position_embeddings
|
||||
q, k = apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=0)
|
||||
|
||||
if USE_XFORMERS_OPS:
|
||||
# Transpose q and k back for attention
|
||||
q = q.transpose(1, 2).contiguous()
|
||||
k = k.transpose(1, 2).contiguous()
|
||||
out = xops.memory_efficient_attention(q, k, v, attn_bias=attention_mask)
|
||||
else:
|
||||
v = v.transpose(1, 2)
|
||||
out = nn.functional.scaled_dot_product_attention(
|
||||
q, k, v, attn_mask=attention_mask
|
||||
)
|
||||
out = out.transpose(1, 2)
|
||||
out = self.attn(
|
||||
q.transpose(1, 2).contiguous(),
|
||||
k.transpose(1, 2).contiguous(),
|
||||
v,
|
||||
cu_seqlens=cu_seqlens,
|
||||
max_seqlen=max_seqlen,
|
||||
sequence_lengths=sequence_lengths,
|
||||
)
|
||||
|
||||
out = out.reshape(batch, patches, self.n_heads * self.head_dim)
|
||||
attn_output, _ = self.o_proj(out)
|
||||
@@ -1307,13 +1349,17 @@ class PixtralHFTransformerBlock(nn.Module):
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
attention_mask: torch.Tensor,
|
||||
position_embeddings: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
max_seqlen: torch.Tensor,
|
||||
sequence_lengths: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
r, _ = self.attention.forward(
|
||||
self.attention_norm(hidden_states),
|
||||
attention_mask=attention_mask,
|
||||
position_embeddings=position_embeddings,
|
||||
cu_seqlens=cu_seqlens,
|
||||
max_seqlen=max_seqlen,
|
||||
sequence_lengths=sequence_lengths,
|
||||
)
|
||||
h = hidden_states + r
|
||||
r = self.feed_forward.forward(self.ffn_norm(h))
|
||||
@@ -1351,14 +1397,22 @@ class PixtralHFTransformer(nn.Module):
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
attention_mask: torch.Tensor,
|
||||
position_embeddings: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
max_seqlen: torch.Tensor,
|
||||
sequence_lengths: torch.Tensor | None,
|
||||
return_all_hidden_states: bool,
|
||||
) -> torch.Tensor:
|
||||
hidden_states_pool = [x]
|
||||
|
||||
for layer in self.layers:
|
||||
x = layer(x, attention_mask, position_embeddings)
|
||||
x = layer(
|
||||
x,
|
||||
position_embeddings,
|
||||
cu_seqlens,
|
||||
max_seqlen,
|
||||
sequence_lengths,
|
||||
)
|
||||
if return_all_hidden_states:
|
||||
hidden_states_pool.append(x)
|
||||
# If we have multiple feature sample layers, we return all hidden
|
||||
@@ -1453,23 +1507,21 @@ class PixtralHFVisionModel(nn.Module):
|
||||
).to(self.device)
|
||||
position_embedding = self.patch_positional_embedding(patch_embeds, position_ids)
|
||||
|
||||
if USE_XFORMERS_OPS:
|
||||
attention_mask = xops.fmha.attn_bias.BlockDiagonalMask.from_seqlens(
|
||||
[p.shape[-2] * p.shape[-1] for p in patch_embeds_list],
|
||||
)
|
||||
else:
|
||||
from transformers.models.pixtral.modeling_pixtral import (
|
||||
generate_block_attention_mask,
|
||||
)
|
||||
|
||||
attention_mask = generate_block_attention_mask(
|
||||
[p.shape[-2] * p.shape[-1] for p in patch_embeds_list], patch_embeds
|
||||
)
|
||||
attention = self.transformer.layers[0].attention.attn
|
||||
cu_seqlens, max_seqlen, sequence_lengths = _make_packed_sequence_metadata(
|
||||
embed_sizes,
|
||||
attention.attn_backend,
|
||||
self.config.hidden_size,
|
||||
self.transformer.layers[0].attention.tp_size,
|
||||
patch_embeds.device,
|
||||
)
|
||||
|
||||
out = self.transformer(
|
||||
patch_embeds,
|
||||
attention_mask,
|
||||
position_embedding,
|
||||
cu_seqlens,
|
||||
max_seqlen,
|
||||
sequence_lengths,
|
||||
return_all_hidden_states=select_layers is not None,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user