[MM][Mistral] Add compile support for Pixtral vision encoders (#57168)

Signed-off-by: Jakub Zakrzewski <jzakrzewski@nvidia.com>
Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Jakub Zakrzewski
2026-09-30 20:27:12 +08:00
committed by GitHub
co-authored by Nicolò Lucchesi mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 93dd80f636
commit effdea9a53
2 changed files with 108 additions and 6 deletions
+33 -5
View File
@@ -21,6 +21,10 @@ from transformers.models.pixtral.image_processing_pixtral import (
)
from transformers.models.pixtral.modeling_pixtral import apply_rotary_pos_emb
from vllm.compilation.decorators import (
should_torch_compile_mm_encoder,
support_torch_compile,
)
from vllm.config import VllmConfig
from vllm.config.multimodal import MultiModalDummyOptions
from vllm.distributed import divide, get_tensor_model_parallel_world_size
@@ -829,6 +833,16 @@ class Attention(nn.Module):
return out
@support_torch_compile(
dynamic_arg_dims={
"x": 1,
"freqs_cis": 0,
"cu_seqlens": 0,
"sequence_lengths": 0,
},
enable_if=should_torch_compile_mm_encoder,
is_encoder=True,
)
class TransformerBlock(nn.Module):
def __init__(
self,
@@ -1298,7 +1312,7 @@ class PixtralHFAttention(nn.Module):
def forward(
self,
hidden_states: torch.Tensor,
position_embeddings: torch.Tensor,
position_embeddings: tuple[torch.Tensor, torch.Tensor],
cu_seqlens: torch.Tensor,
max_seqlen: torch.Tensor,
sequence_lengths: torch.Tensor | None,
@@ -1329,6 +1343,17 @@ class PixtralHFAttention(nn.Module):
return attn_output, None
@support_torch_compile(
dynamic_arg_dims={
"hidden_states": 1,
"rotary_pos_emb_cos": 0,
"rotary_pos_emb_sin": 0,
"cu_seqlens": 0,
"sequence_lengths": 0,
},
enable_if=should_torch_compile_mm_encoder,
is_encoder=True,
)
class PixtralHFTransformerBlock(nn.Module):
def __init__(
self,
@@ -1355,14 +1380,15 @@ class PixtralHFTransformerBlock(nn.Module):
def forward(
self,
hidden_states: torch.Tensor,
position_embeddings: torch.Tensor,
rotary_pos_emb_cos: torch.Tensor,
rotary_pos_emb_sin: torch.Tensor,
cu_seqlens: torch.Tensor,
max_seqlen: torch.Tensor,
sequence_lengths: torch.Tensor | None,
) -> torch.Tensor:
r, _ = self.attention.forward(
self.attention_norm(hidden_states),
position_embeddings=position_embeddings,
position_embeddings=(rotary_pos_emb_cos, rotary_pos_emb_sin),
cu_seqlens=cu_seqlens,
max_seqlen=max_seqlen,
sequence_lengths=sequence_lengths,
@@ -1403,18 +1429,20 @@ class PixtralHFTransformer(nn.Module):
def forward(
self,
x: torch.Tensor,
position_embeddings: torch.Tensor,
position_embeddings: tuple[torch.Tensor, torch.Tensor],
cu_seqlens: torch.Tensor,
max_seqlen: torch.Tensor,
sequence_lengths: torch.Tensor | None,
return_all_hidden_states: bool,
) -> torch.Tensor:
hidden_states_pool = [x]
rotary_pos_emb_cos, rotary_pos_emb_sin = position_embeddings
for layer in self.layers:
x = layer(
x,
position_embeddings,
rotary_pos_emb_cos,
rotary_pos_emb_sin,
cu_seqlens,
max_seqlen,
sequence_lengths,