[MM][Mistral] Add compile support for Pixtral vision encoders (#57168)
Signed-off-by: Jakub Zakrzewski <jzakrzewski@nvidia.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Nicolò Lucchesi
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
93dd80f636
commit
effdea9a53
@@ -21,6 +21,10 @@ from transformers.models.pixtral.image_processing_pixtral import (
|
||||
)
|
||||
from transformers.models.pixtral.modeling_pixtral import apply_rotary_pos_emb
|
||||
|
||||
from vllm.compilation.decorators import (
|
||||
should_torch_compile_mm_encoder,
|
||||
support_torch_compile,
|
||||
)
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.multimodal import MultiModalDummyOptions
|
||||
from vllm.distributed import divide, get_tensor_model_parallel_world_size
|
||||
@@ -829,6 +833,16 @@ class Attention(nn.Module):
|
||||
return out
|
||||
|
||||
|
||||
@support_torch_compile(
|
||||
dynamic_arg_dims={
|
||||
"x": 1,
|
||||
"freqs_cis": 0,
|
||||
"cu_seqlens": 0,
|
||||
"sequence_lengths": 0,
|
||||
},
|
||||
enable_if=should_torch_compile_mm_encoder,
|
||||
is_encoder=True,
|
||||
)
|
||||
class TransformerBlock(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -1298,7 +1312,7 @@ class PixtralHFAttention(nn.Module):
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
position_embeddings: torch.Tensor,
|
||||
position_embeddings: tuple[torch.Tensor, torch.Tensor],
|
||||
cu_seqlens: torch.Tensor,
|
||||
max_seqlen: torch.Tensor,
|
||||
sequence_lengths: torch.Tensor | None,
|
||||
@@ -1329,6 +1343,17 @@ class PixtralHFAttention(nn.Module):
|
||||
return attn_output, None
|
||||
|
||||
|
||||
@support_torch_compile(
|
||||
dynamic_arg_dims={
|
||||
"hidden_states": 1,
|
||||
"rotary_pos_emb_cos": 0,
|
||||
"rotary_pos_emb_sin": 0,
|
||||
"cu_seqlens": 0,
|
||||
"sequence_lengths": 0,
|
||||
},
|
||||
enable_if=should_torch_compile_mm_encoder,
|
||||
is_encoder=True,
|
||||
)
|
||||
class PixtralHFTransformerBlock(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -1355,14 +1380,15 @@ class PixtralHFTransformerBlock(nn.Module):
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
position_embeddings: torch.Tensor,
|
||||
rotary_pos_emb_cos: torch.Tensor,
|
||||
rotary_pos_emb_sin: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
max_seqlen: torch.Tensor,
|
||||
sequence_lengths: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
r, _ = self.attention.forward(
|
||||
self.attention_norm(hidden_states),
|
||||
position_embeddings=position_embeddings,
|
||||
position_embeddings=(rotary_pos_emb_cos, rotary_pos_emb_sin),
|
||||
cu_seqlens=cu_seqlens,
|
||||
max_seqlen=max_seqlen,
|
||||
sequence_lengths=sequence_lengths,
|
||||
@@ -1403,18 +1429,20 @@ class PixtralHFTransformer(nn.Module):
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
position_embeddings: torch.Tensor,
|
||||
position_embeddings: tuple[torch.Tensor, torch.Tensor],
|
||||
cu_seqlens: torch.Tensor,
|
||||
max_seqlen: torch.Tensor,
|
||||
sequence_lengths: torch.Tensor | None,
|
||||
return_all_hidden_states: bool,
|
||||
) -> torch.Tensor:
|
||||
hidden_states_pool = [x]
|
||||
rotary_pos_emb_cos, rotary_pos_emb_sin = position_embeddings
|
||||
|
||||
for layer in self.layers:
|
||||
x = layer(
|
||||
x,
|
||||
position_embeddings,
|
||||
rotary_pos_emb_cos,
|
||||
rotary_pos_emb_sin,
|
||||
cu_seqlens,
|
||||
max_seqlen,
|
||||
sequence_lengths,
|
||||
|
||||
Reference in New Issue
Block a user