[V1][VLM][Pixtral-HF] Support Pixtral-HF on V1 (#14275)

Signed-off-by: Linkun Chen <github@lkchen.net>
This commit is contained in:
lkchen
2025-03-06 08:58:41 +00:00
committed by GitHub
parent 1769928079
commit 5d802522a7
4 changed files with 175 additions and 16 deletions
+8 -2
View File
@@ -1042,9 +1042,13 @@ class PixtralHFVisionModel(nn.Module):
for img in pixel_values
]
patch_embeds = [
p.flatten(2).permute(0, 2, 1) for p in patch_embeds_list
]
embed_sizes = [p.shape[1] for p in patch_embeds]
# flatten to a single sequence
patch_embeds = torch.cat(
[p.flatten(2).permute(0, 2, 1) for p in patch_embeds_list], dim=1)
patch_embeds = torch.cat(patch_embeds, dim=1)
patch_embeds = self.ln_pre(patch_embeds)
# positional embeddings
@@ -1075,6 +1079,8 @@ class PixtralHFVisionModel(nn.Module):
out = resolve_visual_encoder_outputs(out, feature_sample_layers, None,
self.config.num_hidden_layers)
# squeeze dim 0 and split into separate tensors for each image
out = torch.split(torch.squeeze(out), embed_sizes)
return out
# (TODO) Add prefix argument for filtering out weights to be loaded