[Feature]: Remove DtoH Copy for lfm2_vl On Default Stream (#32815)

Signed-off-by: Tianshu Yu <tianshuyu.formal@gmail.com>
This commit is contained in:
tianshu-Michael-yu
2026-01-23 13:20:30 +00:00
committed by GitHub
parent 10e94c84f6
commit 13d8746c54
5 changed files with 260 additions and 158 deletions
+128 -50
View File
@@ -50,7 +50,7 @@ from .interfaces import (
SupportsMultiModal,
SupportsPP,
)
from .siglip2 import Siglip2Model
from .lfm2_siglip2 import Siglip2Model
from .utils import (
AutoWeightsLoader,
WeightsMapper,
@@ -450,29 +450,78 @@ class Lfm2VLMultiModalProjector(nn.Module):
bias=config.projector_bias,
)
def forward(self, image_features: torch.Tensor):
image_features = self.pixel_unshuffle(image_features)
if self.projector_use_layernorm:
image_features = self.layer_norm(image_features)
hidden_states = self.linear_1(image_features)
hidden_states = self.act(hidden_states)
hidden_states = self.linear_2(hidden_states)
return hidden_states
def forward(
self,
vision_features_packed: torch.Tensor,
spatial_shapes: torch.Tensor,
) -> torch.Tensor:
"""Project packed vision features without materializing padded tensors.
def pixel_unshuffle(self, hidden_states: torch.Tensor):
batch_size, width, height, channels = hidden_states.size()
hidden_states = hidden_states.reshape(
batch_size, width, height // self.factor, channels * self.factor
Args:
vision_features_packed: (total_tokens, hidden_size) packed in tile order.
spatial_shapes: (num_tiles, 2) on CPU (height, width) per tile.
Returns:
projected_packed: (total_projected_tokens, text_hidden_size)
"""
assert spatial_shapes.device.type == "cpu", (
"Expected `spatial_shapes` on CPU to avoid device-to-host sync in "
"variable-length packing."
)
hidden_states = hidden_states.permute(0, 2, 1, 3)
hidden_states = hidden_states.reshape(
batch_size,
height // self.factor,
width // self.factor,
channels * self.factor**2,
)
hidden_states = hidden_states.permute(0, 2, 1, 3)
return hidden_states
factor = self.factor
device = vision_features_packed.device
hidden_size = vision_features_packed.shape[-1]
spatial_shapes_list: list[list[int]] = spatial_shapes.tolist()
lengths_list = [h * w for h, w in spatial_shapes_list]
gather_idx_parts: list[torch.Tensor] = []
offset = 0
dh = torch.arange(factor, dtype=torch.int64)
dw = torch.arange(factor, dtype=torch.int64)
dh_grid, dw_grid = torch.meshgrid(dh, dw, indexing="ij")
dh_flat = dh_grid.reshape(-1)
dw_flat = dw_grid.reshape(-1)
for (height, width), length in zip(spatial_shapes_list, lengths_list):
if length <= 0:
continue
if height % factor != 0 or width % factor != 0:
raise ValueError(
"spatial_shapes must be divisible by downsample_factor: "
f"got ({height}, {width}) with factor={factor}."
)
height_out = height // factor
width_out = width // factor
rows_out = torch.arange(height_out, dtype=torch.int64)
cols_out = torch.arange(width_out, dtype=torch.int64)
rr, cc = torch.meshgrid(rows_out, cols_out, indexing="ij")
rr = rr.reshape(-1)
cc = cc.reshape(-1)
token_idx = (rr[:, None] * factor + dh_flat[None, :]) * width + (
cc[:, None] * factor + dw_flat[None, :]
)
gather_idx_parts.append(token_idx.reshape(-1) + offset)
offset += length
if gather_idx_parts:
gather_idx = torch.cat(gather_idx_parts).to(device=device)
gathered = vision_features_packed.index_select(0, gather_idx)
unshuffled = gathered.reshape(-1, factor * factor * hidden_size)
else:
unshuffled = vision_features_packed.new_empty(
(0, factor * factor * hidden_size)
)
if self.projector_use_layernorm:
unshuffled = self.layer_norm(unshuffled)
hidden_states = self.linear_1(unshuffled)
hidden_states = self.act(hidden_states)
projected_packed = self.linear_2(hidden_states)
return projected_packed
@MULTIMODAL_REGISTRY.register_processor(
@@ -598,61 +647,90 @@ class Lfm2VLForConditionalGeneration(
pixel_values: torch.FloatTensor,
spatial_shapes: torch.Tensor,
) -> torch.Tensor:
assert spatial_shapes.device.type == "cpu", (
"Expected `spatial_shapes` on CPU to avoid device-to-host sync in "
"variable-length packing."
)
pixel_values = pixel_values.to(
dtype=self.vision_tower.vision_model.embeddings.patch_embedding.weight.dtype
) # fp16 compatibility
# LFM2-VL's HF processor pads patch sequences with trailing zeros.
# Derive the valid-patch mask from spatial_shapes instead of carrying
# pixel_attention_mask through the vLLM multimodal pipeline.
max_seq_len = pixel_values.shape[1]
# Pack patch tokens upfront so the vision tower runs entirely unpadded.
spatial_shapes_list: list[list[int]] = spatial_shapes.tolist()
lengths_list = [h * w for h, w in spatial_shapes_list]
total_tokens = int(sum(lengths_list))
lengths_cpu = (spatial_shapes[:, 0] * spatial_shapes[:, 1]).to(
dtype=torch.int32
)
max_seqlen = (
lengths_cpu.max().reshape(1).to(device=pixel_values.device)
lengths_cpu.max().reshape(1)
if lengths_cpu.numel()
else torch.tensor([0], dtype=torch.int32, device=pixel_values.device)
else torch.tensor([0], dtype=torch.int32)
)
lengths = lengths_cpu.to(device=pixel_values.device)
packed_mask = (
torch.arange(max_seq_len, device=pixel_values.device)[None, :]
< lengths[:, None]
if total_tokens == 0:
return []
packed_pixel_values = pixel_values.new_empty(
(total_tokens, pixel_values.shape[-1])
)
offset = 0
for i, length in enumerate(lengths_list):
if length <= 0:
continue
packed_pixel_values[offset : offset + length].copy_(
pixel_values[i, :length]
)
offset += length
packed_pixel_values = packed_pixel_values.unsqueeze(0)
lengths = torch.tensor(
lengths_list, dtype=torch.int32, device=pixel_values.device
)
cu_seqlens = torch.zeros(
lengths.shape[0] + 1,
dtype=torch.int32,
device=lengths.device,
device=pixel_values.device,
)
cu_seqlens[1:] = torch.cumsum(lengths, dim=0)
with set_forward_context(None, self.vllm_config):
vision_outputs = self.vision_tower(
pixel_values=pixel_values,
pixel_values_packed=packed_pixel_values,
spatial_shapes=spatial_shapes,
packed_mask=packed_mask,
cu_seqlens=cu_seqlens,
max_seqlen=max_seqlen,
)
image_outputs = getattr(vision_outputs, "last_hidden_state", vision_outputs)
image_outputs_packed = getattr(
vision_outputs, "last_hidden_state", vision_outputs
)
vision_features_packed = image_outputs_packed[0]
image_features = []
factor = self.multi_modal_projector.factor
projected_lengths_list: list[int] = []
for (height, width), length in zip(spatial_shapes_list, lengths_list):
if length <= 0:
projected_lengths_list.append(0)
continue
if height % factor != 0 or width % factor != 0:
raise ValueError(
"spatial_shapes must be divisible by downsample_factor: "
f"got ({height}, {width}) with factor={factor}."
)
projected_lengths_list.append((height // factor) * (width // factor))
# spatial_shapes is on CPU (keep_on_cpu=True), so .tolist() is instant
spatial_shapes_list = spatial_shapes.tolist()
for img_idx, (feature_org_h, feature_org_w) in enumerate(spatial_shapes_list):
feature_len = feature_org_h * feature_org_w
feature = image_outputs[img_idx, :feature_len]
projected_packed = self.multi_modal_projector(
vision_features_packed=vision_features_packed,
spatial_shapes=spatial_shapes,
)
# reshape to original height and width
feature = feature.reshape(1, feature_org_h, feature_org_w, -1)
# project the image representation
img_embedding = self.multi_modal_projector(feature)
# flatten here to handle variable length in naflex
img_embedding = img_embedding.reshape(-1, img_embedding.size(-1))
image_features.append(img_embedding)
image_features: list[torch.Tensor] = []
offset = 0
for out_len in projected_lengths_list:
image_features.append(projected_packed[offset : offset + out_len])
offset += out_len
return image_features