[Core] Check for GPU<->CPU syncs during CI (#43107)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Nick Hill
2026-08-14 17:42:05 -07:00
committed by GitHub
co-authored by Claude Opus 4.8
parent 7b544ecb52
commit 615d4cfade
44 changed files with 600 additions and 357 deletions
+9 -5
View File
@@ -54,6 +54,7 @@ from vllm.multimodal.processing import (
PromptUpdateDetails,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from .interfaces import (
@@ -306,7 +307,10 @@ def _flatten_valid_audio_embeddings(
< output_lengths[:, None]
)
return audio_embeddings[valid_mask], output_lengths
# Boolean-mask indexing has a data-dependent output shape, so the count
# has to come back to the host.
with gpu_sync_allowed():
return audio_embeddings[valid_mask], output_lengths
def _count_audio_tokens_from_mask(
@@ -630,10 +634,10 @@ class AudioFlamingo3ForConditionalGeneration(
audio_features,
feature_attention_mask,
)
chunk_embeddings = torch.split(
masked_audio_features,
audio_output_lengths.tolist(),
)
# `split` needs Python int sizes.
with gpu_sync_allowed():
split_sizes = audio_output_lengths.tolist()
chunk_embeddings = torch.split(masked_audio_features, split_sizes)
grouped_embeddings = []
current_idx = 0
+4 -4
View File
@@ -56,6 +56,7 @@ from vllm.transformers_utils.processors.cohere_asr import (
CohereASRProcessor,
)
from vllm.utils.collection_utils import is_list_of
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.v1.attention.backend import (
AttentionType,
)
@@ -1770,10 +1771,9 @@ class CohereASRModel(nn.Module):
out = self.encoder_decoder_proj(out)
# Convert padded tensor to packed
outs = []
for i, feat in enumerate(out):
feat_len = encoder_output_length[i]
outs.append(feat[:feat_len, :])
with gpu_sync_allowed():
lengths = encoder_output_length.tolist()
outs = [feat[:length, :] for feat, length in zip(out, lengths)]
return outs
else:
+13 -3
View File
@@ -71,6 +71,7 @@ from vllm.multimodal.processing import (
PromptUpdate,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backends.registry import AttentionBackendEnum
@@ -788,7 +789,10 @@ class VariableResolutionResamplerModel(nn.Module):
return x
def fwd_placeholder(x, grid_thw, to_tensor=False):
grid_thw_cpu = grid_thw.cpu().numpy()
# The per-image grids drive the Python-level offset arithmetic
# below; the same read is wrapped on the cudagraph path.
with gpu_sync_allowed():
grid_thw_cpu = grid_thw.cpu().numpy()
grid_t, grid_hw = grid_thw_cpu[:, 0], grid_thw_cpu[:, 1:]
grid_hw_after_conv = grid_hw.prod(-1) // (self.spatial_conv_size**2)
@@ -1697,8 +1701,11 @@ class Ernie4_5_VLMoeForConditionalGeneration(
EncoderCudaGraphReplayBuffers,
)
# The per-image grids are needed as Python ints to size the buffers.
with gpu_sync_allowed():
grid_thw_list = mm_kwargs["image_grid_thw"].tolist()
metadata = self.vision_model.prepare_encoder_metadata(
mm_kwargs["image_grid_thw"].tolist(), max_batch_size=max_batch_size
grid_thw_list, max_batch_size=max_batch_size
)
values = metadata | {
"pixel_values": mm_kwargs["pixel_values"],
@@ -1738,7 +1745,10 @@ class Ernie4_5_VLMoeForConditionalGeneration(
# Ernie only uses the single "default" encoder path.
output = outputs["default"]
grid_thw = batch_mm_kwargs["image_grid_thw"].to(output.device)
num_valid = int((grid_thw[:, 0] * grid_thw[:, 1] * grid_thw[:, 2]).sum())
# The valid token count slices the graph output for the eager
# resampler call, so it has to come back to the host.
with gpu_sync_allowed():
num_valid = int((grid_thw[:, 0] * grid_thw[:, 1] * grid_thw[:, 2]).sum())
image_embeds = self.resampler_model(output[:num_valid], grid_thw)
scatter_output_slices(image_embeds, indices, per_item_out_tokens, dest, clone)
+43 -27
View File
@@ -69,6 +69,7 @@ from vllm.multimodal.processing.processor import (
PromptUpdateDetails,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.utils.torch_utils import async_tensor_h2d
@@ -1365,11 +1366,14 @@ class Gemma4ForConditionalGeneration(
pp_tensor,
pad_tensor,
).to(self.model_dtype)
encoder_outputs = vt.encoder(
inputs_embeds=inputs_embeds,
attention_mask=~pad_tensor,
pixel_position_ids=pp_tensor,
)
# HuggingFace's mask builder probes `padding_mask.all()` to
# decide whether the mask can be skipped, which syncs.
with gpu_sync_allowed():
encoder_outputs = vt.encoder(
inputs_embeds=inputs_embeds,
attention_mask=~pad_tensor,
pixel_position_ids=pp_tensor,
)
hidden_states = encoder_outputs.last_hidden_state
for i, (orig_idx, _, _) in enumerate(chunk_items):
@@ -1387,13 +1391,17 @@ class Gemma4ForConditionalGeneration(
single_pos_ids = pool_position_ids[orig_idx].unsqueeze(0)
padding_positions = (single_pos_ids == -1).all(dim=-1)
pooled_states, valid_mask = vt.pooler(
hidden_states=single_hidden,
pixel_position_ids=single_pos_ids,
padding_positions=padding_positions,
output_length=output_length,
)
valid_states = pooled_states[valid_mask]
# The pooler goes through HuggingFace's mask builder, which probes
# `padding_mask.all()`, and the mask indexing below needs the
# selected count on the host.
with gpu_sync_allowed():
pooled_states, valid_mask = vt.pooler(
hidden_states=single_hidden,
pixel_position_ids=single_pos_ids,
padding_positions=padding_positions,
output_length=output_length,
)
valid_states = pooled_states[valid_mask]
if getattr(vt.config, "standardize", False):
valid_states = (valid_states - vt.std_bias) * vt.std_scale
@@ -1444,7 +1452,9 @@ class Gemma4ForConditionalGeneration(
pooling_k2 = vision_cfg.pooling_kernel_size**2
if isinstance(frame_counts, torch.Tensor):
fc_list = frame_counts.tolist()
# Per-video frame counts drive the Python-level batching below.
with gpu_sync_allowed():
fc_list = frame_counts.tolist()
else:
fc_list = list(frame_counts)
@@ -1474,11 +1484,13 @@ class Gemma4ForConditionalGeneration(
pp_chunk,
pad_chunk,
).to(self.model_dtype)
encoder_outputs = vt.encoder(
inputs_embeds=inputs_embeds,
attention_mask=~pad_chunk,
pixel_position_ids=pp_chunk,
)
# HuggingFace's mask builder probes `padding_mask.all()`.
with gpu_sync_allowed():
encoder_outputs = vt.encoder(
inputs_embeds=inputs_embeds,
attention_mask=~pad_chunk,
pixel_position_ids=pp_chunk,
)
last_hidden_states_list.append(encoder_outputs.last_hidden_state)
last_hidden_states = torch.cat(last_hidden_states_list, dim=0)
@@ -1493,13 +1505,15 @@ class Gemma4ForConditionalGeneration(
single_pos_ids = pixel_position_ids[i].unsqueeze(0)
single_pad_pos = padding_positions[i].unsqueeze(0)
pooled_states, valid_mask = vt.pooler(
hidden_states=single_hidden,
pixel_position_ids=single_pos_ids,
padding_positions=single_pad_pos,
output_length=output_length,
)
valid_states = pooled_states[valid_mask]
# As above, plus mask indexing that needs the count on the host.
with gpu_sync_allowed():
pooled_states, valid_mask = vt.pooler(
hidden_states=single_hidden,
pixel_position_ids=single_pos_ids,
padding_positions=single_pad_pos,
output_length=output_length,
)
valid_states = pooled_states[valid_mask]
if getattr(vt.config, "standardize", False):
valid_states = (valid_states - vt.std_bias) * vt.std_scale
@@ -1553,9 +1567,11 @@ class Gemma4ForConditionalGeneration(
# Strip padding per-batch element: only keep valid (non-padding)
# tokens.
# Boolean-mask indexing needs the selected count on the host.
per_audio = []
for enc, mask in zip(audio_features, audio_mask, strict=True):
per_audio.append(enc[mask]) # [num_real, hidden_size]
with gpu_sync_allowed():
for enc, mask in zip(audio_features, audio_mask, strict=True):
per_audio.append(enc[mask]) # [num_real, hidden_size]
return per_audio
+4 -2
View File
@@ -60,6 +60,7 @@ from vllm.multimodal.processing import (
from vllm.sequence import IntermediateTensors
from vllm.tokenizers import cached_tokenizer_from_config
from vllm.transformers_utils.processor import cached_processor_from_config
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.utils.torch_utils import PIN_MEMORY
@@ -788,8 +789,9 @@ class GraniteSpeechForConditionalGeneration(
encoder_embeds = self.encoder(audio_input["input_features"])
# [bsz, <max feature size>, 4096]
projected_embeds = self.projector(encoder_embeds)
# Apply mask on variable length audio features
masked_embeds = projected_embeds[audio_input["input_features_mask"]]
# Apply mask on variable length audio features.
with gpu_sync_allowed():
masked_embeds = projected_embeds[audio_input["input_features_mask"]]
# Split variable length features into a tuple
return torch.split(masked_embeds, audio_input["audio_embed_sizes"])
@@ -39,6 +39,7 @@ from vllm.model_executor.layers.linear import (
RowParallelLinear,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from .utils import AutoWeightsLoader, WeightsMapper
from .vision import is_vit_use_data_parallel, run_dp_sharded_vision_model
@@ -96,10 +97,14 @@ class Idefics2VisionEmbeddings(nn.Module):
size=(batch_size, max_nb_patches_h * max_nb_patches_w), fill_value=0
)
for batch_idx, p_attn_mask in enumerate(patch_attention_mask):
if tgt_sizes is not None:
nb_patches_h = tgt_sizes[batch_idx][0]
nb_patches_w = tgt_sizes[batch_idx][1]
with gpu_sync_allowed():
patch_attention_mask_cpu = patch_attention_mask.cpu()
tgt_sizes_cpu = tgt_sizes.cpu() if tgt_sizes is not None else None
for batch_idx, p_attn_mask in enumerate(patch_attention_mask_cpu):
if tgt_sizes_cpu is not None:
nb_patches_h = tgt_sizes_cpu[batch_idx][0]
nb_patches_w = tgt_sizes_cpu[batch_idx][1]
else:
nb_patches_h = p_attn_mask[:, 0].sum()
nb_patches_w = p_attn_mask[0].sum()
@@ -114,8 +119,10 @@ class Idefics2VisionEmbeddings(nn.Module):
pos_ids = (
bucket_coords_h[:, None] * self.num_patches_per_side + bucket_coords_w
).flatten()
position_ids[batch_idx][p_attn_mask.view(-1).cpu()] = pos_ids
position_ids = position_ids.to(self.position_embedding.weight.device)
position_ids[batch_idx][p_attn_mask.view(-1)] = pos_ids
position_ids = position_ids.to(
self.position_embedding.weight.device, non_blocking=True
)
embeddings += self.position_embedding(position_ids)
return embeddings
+6 -4
View File
@@ -51,6 +51,7 @@ from vllm.multimodal.processing import (
PromptUpdateDetails,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from .idefics2_vision_model import (
@@ -443,11 +444,12 @@ class Idefics3Model(nn.Module):
real_images_inds = (pixel_values == 0.0).sum(
dim=(-1, -2, -3)
) != nb_values_per_image
pixel_values = pixel_values[real_images_inds].contiguous()
with gpu_sync_allowed():
pixel_values = pixel_values[real_images_inds].contiguous()
# Handle the vision attention mask
# Remove padding images from the mask
pixel_attention_mask = pixel_attention_mask[real_images_inds].contiguous()
# Handle the vision attention mask
# Remove padding images from the mask
pixel_attention_mask = pixel_attention_mask[real_images_inds].contiguous()
patch_size = self.config.vision_config.patch_size
patches_subgrid = pixel_attention_mask.unfold(
+18 -12
View File
@@ -16,6 +16,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.sequence import IntermediateTensors
from vllm.tasks import PoolingTask
from vllm.transformers_utils.repo_utils import get_hf_file_bytes
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.v1.pool.metadata import PoolingMetadata
from ..layers.pooler import DispatchPooler
@@ -110,19 +111,24 @@ class JinaForRankingPool(StepPool):
prompt_token_ids = pooling_metadata.get_prompt_token_ids()
embeds_list = list[torch.Tensor | None]()
for data, token_ids in zip(pooled_data_lst, prompt_token_ids):
# for unfinished chunked prefill
if data is None:
embeds_list.append(None)
else:
docs_indexes = torch.where(torch.eq(token_ids, self.doc_token_id))[0]
query_indexes = torch.where(torch.eq(token_ids, self.query_token_id))[0]
# `torch.where` resolves the match count on the host.
with gpu_sync_allowed():
for data, token_ids in zip(pooled_data_lst, prompt_token_ids):
# for unfinished chunked prefill
if data is None:
embeds_list.append(None)
else:
doc_match = torch.eq(token_ids, self.doc_token_id)
docs_indexes = torch.where(doc_match)[0]
query_indexes = torch.where(
torch.eq(token_ids, self.query_token_id)
)[0]
# The JinaForRanking model concatenates docs first, then query.
# Let's stay consistent with this novel design.
indexes = torch.cat([docs_indexes, query_indexes])
embeds = self.projector(data[indexes])
embeds_list.append(embeds)
# The JinaForRanking model concatenates docs first, then query.
# Let's stay consistent with this novel design.
indexes = torch.cat([docs_indexes, query_indexes])
embeds = self.projector(data[indexes])
embeds_list.append(embeds)
return embeds_list
+10 -4
View File
@@ -98,6 +98,7 @@ from vllm.transformers_utils.processor import (
)
from vllm.transformers_utils.utils import convert_model_repo_to_path
from vllm.utils.collection_utils import flatten_2d_lists
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.utils.torch_utils import set_default_torch_dtype
@@ -1548,7 +1549,8 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
all_pixel_values[i, ..., :L_item] = pixel_values_item
num_patches = tgt_sizes.prod(-1)
max_patches = num_patches.max().item()
with gpu_sync_allowed():
max_patches = num_patches.max().item()
assert isinstance(max_patches, int)
patch_attn_mask = torch.zeros((B, max_patches), dtype=torch.bool, device=device)
@@ -1640,7 +1642,9 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
all_pixel_values[i, ..., :L_item] = pixel_values_item
num_patches = tgt_sizes.prod(-1)
max_patches = num_patches.max().item()
# Needed as a Python int to size the mask below.
with gpu_sync_allowed():
max_patches = num_patches.max().item()
assert isinstance(max_patches, int)
patch_attn_mask = torch.zeros((B, max_patches), dtype=torch.bool, device=device)
@@ -1737,7 +1741,8 @@ class MiniCPMV4_0(MiniCPMVBaseModel, SupportsLoRA):
all_pixel_values[i, ..., :L_item] = pixel_values_item
num_patches = tgt_sizes.prod(-1)
max_patches = num_patches.max().item()
with gpu_sync_allowed():
max_patches = num_patches.max().item()
assert isinstance(max_patches, int)
patch_attn_mask = torch.zeros((B, max_patches), dtype=torch.bool, device=device)
@@ -1839,7 +1844,8 @@ class MiniCPMV4_5(MiniCPMVBaseModel, SupportsLoRA):
all_pixel_values[i, ..., :L_item] = pixel_values_item
num_patches = tgt_sizes.prod(-1)
max_patches = num_patches.max().item()
with gpu_sync_allowed():
max_patches = num_patches.max().item()
assert isinstance(max_patches, int)
patch_attn_mask = torch.zeros((B, max_patches), dtype=torch.bool, device=device)
+11 -5
View File
@@ -66,6 +66,7 @@ from vllm.model_executor.models.utils import maybe_prefix
from vllm.model_executor.models.vision import is_vit_use_data_parallel
from vllm.platforms import current_platform
from vllm.transformers_utils.configs.moonvit import MoonViTConfig
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.torch_utils import async_tensor_h2d
@@ -811,9 +812,14 @@ class MoonVitPretrainedModel(PreTrainedModel):
merge_kernel_size=self.merge_kernel_size,
)
hidden_states = self.patch_embed(pixel_values, grid_hw)
hidden_states = self.encoder(hidden_states, grid_hw)
hidden_states = patch_merger(
hidden_states, grid_hw, merge_kernel_size=self.merge_kernel_size
)
# Legacy path: patch_embed, encoder and patch_merger each iterate the
# per-image grids in Python, so all three read `grid_hw` back to the
# host. The `encoder_metadata` path above precomputes that outside the
# graph and does not sync.
with gpu_sync_allowed():
hidden_states = self.patch_embed(pixel_values, grid_hw)
hidden_states = self.encoder(hidden_states, grid_hw)
hidden_states = patch_merger(
hidden_states, grid_hw, merge_kernel_size=self.merge_kernel_size
)
return hidden_states
+14 -9
View File
@@ -55,6 +55,7 @@ from vllm.multimodal.processing import (
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from .interfaces import (
@@ -1667,15 +1668,19 @@ class MossAudioModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
"""
audio_data = audio_input["audio_data"]
audio_data_seqlens = audio_input["audio_data_seqlens"]
last_hidden_state, deepstack = self.audio_encoder(
audio_data.to(self.audio_encoder.dtype),
feature_lens=audio_data_seqlens,
output_deepstack_hidden_states=len(self.deepstack_audio_merger_list) > 0,
)
audio_embeds = self.audio_adapter(last_hidden_state)
audio_lengths = MossAudioEncoder._compute_downsampled_length(
audio_data_seqlens.to(device=audio_embeds.device, dtype=torch.long)
).tolist()
# The encoder chunks the input by per-audio feature lengths, which
# needs Python ints for `split`/`pad_sequence`.
want_deepstack = len(self.deepstack_audio_merger_list) > 0
with gpu_sync_allowed():
last_hidden_state, deepstack = self.audio_encoder(
audio_data.to(self.audio_encoder.dtype),
feature_lens=audio_data_seqlens,
output_deepstack_hidden_states=want_deepstack,
)
audio_embeds = self.audio_adapter(last_hidden_state)
audio_lengths = MossAudioEncoder._compute_downsampled_length(
audio_data_seqlens.to(device=audio_embeds.device, dtype=torch.long)
).tolist()
main_embeddings = tuple(audio_embeds.squeeze(0).split(audio_lengths, dim=0))
deepstack_embeddings: list[tuple[torch.Tensor, ...]] = []
+15 -3
View File
@@ -49,6 +49,7 @@ from vllm.multimodal.processing.processor import (
ResolvedPromptUpdate,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from .idefics2_vision_model import Idefics2VisionTransformer
@@ -396,12 +397,23 @@ class Phi4MMImageEncoder(nn.Module):
w * base_feat_width // base_feat_height_reduction,
)
)
useful_height = int(reshaped_image_attention_mask[0, :, 0].sum().item())
useful_width = int(reshaped_image_attention_mask[0, 0, :].sum().item())
# The mask stays on device for the encoder above, so these
# per-image counts have to come back to the host to drive the
# slicing and the Python-level length arithmetic.
with gpu_sync_allowed():
useful_height = int(
reshaped_image_attention_mask[0, :, 0].sum().item()
)
useful_width = int(
reshaped_image_attention_mask[0, 0, :].sum().item()
)
mask_token_count = int(
image_attention_mask[_bs, : B_ + 1, 0::2, 0::2].sum().item()
)
sub_img = sub_img[:, :useful_height, :useful_width]
temp_sub_GN = self.sub_GN.repeat(1, useful_height, 1, 1)
temp_len = (
int(image_attention_mask[_bs, : B_ + 1, 0::2, 0::2].sum().item())
mask_token_count
+ (useful_height + 1)
+ base_feat_height // base_feat_height_reduction
)
@@ -89,6 +89,7 @@ from vllm.multimodal.processing.processor import (
PromptUpdateDetails,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from .interfaces import (
@@ -1036,12 +1037,14 @@ class Qwen2_5OmniConditionalGenerationMixin:
self.audio_tower._get_feat_extract_output_lengths(audio_feature_lengths)
)
audio_outputs = self.audio_tower(
input_features.to(self.audio_tower.dtype),
feature_lens=audio_feature_lengths,
aftercnn_lens=audio_feat_lengths,
)
return audio_outputs.last_hidden_state.split(audio_output_lengths.tolist())
with gpu_sync_allowed():
audio_outputs = self.audio_tower(
input_features.to(self.audio_tower.dtype),
feature_lens=audio_feature_lengths,
aftercnn_lens=audio_feat_lengths,
)
split_sizes = audio_output_lengths.tolist()
return audio_outputs.last_hidden_state.split(split_sizes)
def _process_image_input(
self, image_input: Qwen2_5_VLImageInputs
@@ -1526,12 +1529,13 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
video_token_id = self.config.video_token_index
audio_token_id = self.config.audio_token_index
input_ids_cpu = input_ids.cpu()
is_video = is_multimodal & (input_ids_cpu == video_token_id)
is_audio = is_multimodal & (input_ids_cpu == audio_token_id)
with gpu_sync_allowed():
input_ids_cpu = input_ids.cpu()
is_video = is_multimodal & (input_ids_cpu == video_token_id)
is_audio = is_multimodal & (input_ids_cpu == audio_token_id)
num_video = is_video.sum().item()
num_audio = is_audio.sum().item()
num_video = is_video.sum().item()
num_audio = is_audio.sum().item()
if check_interleaved_audio_video(is_video, is_audio, num_video, num_audio):
inputs_embeds = self._embed_text_input_ids(
+14 -11
View File
@@ -87,6 +87,7 @@ from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers
from ...utils.gpu_sync_debug import gpu_sync_allowed
from .interfaces import (
MultiModalEmbeddings,
SupportsEagle,
@@ -1569,8 +1570,9 @@ class Qwen2_5_VLForConditionalGeneration(
video_second_per_grid=video_second_per_grid_t.item(),
).to(emb.device, non_blocking=True)
emb = emb[retention_mask]
positions = positions[retention_mask]
with gpu_sync_allowed():
emb = emb[retention_mask]
positions = positions[retention_mask]
emb = torch.cat([emb, positions], dim=1)
video_embeds_out.append(emb)
return tuple(video_embeds_out)
@@ -1624,15 +1626,16 @@ class Qwen2_5_VLForConditionalGeneration(
mm[:, -4:].permute(1, 0).long() for mm in multimodal_embeddings
]
positions, mrope_positions_delta = recompute_mrope_positions(
input_ids_t,
mm_embeddings_pos,
mrope_positions,
num_computed_tokens,
vision_start_token_id,
image_token_id,
video_token_id,
)
with gpu_sync_allowed():
positions, mrope_positions_delta = recompute_mrope_positions(
input_ids_t,
mm_embeddings_pos,
mrope_positions,
num_computed_tokens,
vision_start_token_id,
image_token_id,
video_token_id,
)
return mm_embeddings_out, positions, mrope_positions_delta
+9 -5
View File
@@ -60,6 +60,7 @@ from vllm.multimodal.processing import (
PromptUpdate,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
@@ -444,12 +445,15 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal, Supports
)
< audio_output_lengths
)
masked_audio_features = audio_features[audio_features_mask].view(-1, embed_dim)
with gpu_sync_allowed():
masked_audio_features = audio_features[audio_features_mask].view(
-1, embed_dim
)
# Split to tuple of embeddings for individual audio input.
return torch.split(
masked_audio_features, audio_output_lengths.flatten().tolist()
)
# Split to tuple of embeddings for individual audio input.
return torch.split(
masked_audio_features, audio_output_lengths.flatten().tolist()
)
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
audio_input = self._parse_and_validate_audio_input(**kwargs)
+4 -1
View File
@@ -92,6 +92,7 @@ from vllm.transformers_utils.processor import cached_processor_from_config
from vllm.transformers_utils.processors.qwen3_asr import (
Qwen3ASRProcessor,
)
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
logger = init_logger(__name__)
_ASR_TEXT_TAG = "<asr_text>"
@@ -467,7 +468,9 @@ class Qwen3ASRForConditionalGeneration(
feature_lens=audio_feature_lengths,
aftercnn_lens=audio_output_lengths,
)
return audio_features.split(audio_output_lengths.tolist())
with gpu_sync_allowed():
split_sizes = audio_output_lengths.tolist()
return audio_features.split(split_sizes)
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings | None:
mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)
@@ -77,6 +77,7 @@ from vllm.multimodal.processing.processor import (
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.processor import cached_processor_from_config
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backends.registry import AttentionBackendEnum
@@ -434,11 +435,14 @@ class Qwen3OmniMoeAudioEncoder(nn.Module):
feature_lens: torch.Tensor,
aftercnn_lens: torch.Tensor,
):
# Compute chunk information
# Compute chunk information.
chunk_num = torch.ceil(feature_lens / (self.n_window * 2)).long()
with gpu_sync_allowed():
total_chunks = int(chunk_num.sum())
chunk_lengths = async_tensor_h2d(
[self.n_window * 2] * chunk_num.sum(),
[self.n_window * 2] * total_chunks,
dtype=torch.long,
device=feature_lens.device,
)
@@ -447,15 +451,18 @@ class Qwen3OmniMoeAudioEncoder(nn.Module):
chunk_lengths[chunk_lengths == 0] = self.n_window * 2
# Split input features into chunks and pad
chunk_list = input_features.T.split(chunk_lengths.tolist(), dim=0)
with gpu_sync_allowed():
chunk_lengths_list = chunk_lengths.tolist()
chunk_list = input_features.T.split(chunk_lengths_list, dim=0)
padded_feature = nn.utils.rnn.pad_sequence(
chunk_list, batch_first=True
).transpose(1, 2)
# Compute feature lengths after CNN
feature_lens_after_cnn = self._get_cnn_output_lengths(chunk_lengths)
# Vectorized mask creation: avoid creating many small tensors
max_len_after_cnn = feature_lens_after_cnn.max().item()
# Vectorized mask creation: avoid creating many small tensors.
with gpu_sync_allowed():
max_len_after_cnn = feature_lens_after_cnn.max().item()
indices = torch.arange(max_len_after_cnn, device=padded_feature.device)
padded_mask_after_cnn = indices.unsqueeze(0) < feature_lens_after_cnn.unsqueeze(
1
@@ -494,16 +501,17 @@ class Qwen3OmniMoeAudioEncoder(nn.Module):
)
padded_embed = padded_embed + positional_embedding
# Extract valid hidden states and compute cu_seqlens
hidden_states = padded_embed[padded_mask_after_cnn]
with gpu_sync_allowed():
hidden_states = padded_embed[padded_mask_after_cnn]
# Use tolist() for efficient batch conversion from tensor to Python.
aftercnn_lens_list = aftercnn_lens.tolist()
# Compute cumulative sequence lengths for chunked attention
cu_chunk_lens = [0]
window_aftercnn = padded_mask_after_cnn.shape[-1] * (
self.n_window_infer // (self.n_window * 2)
)
# Use tolist() for efficient batch conversion from tensor to Python
for cnn_len in aftercnn_lens.tolist():
for cnn_len in aftercnn_lens_list:
num_full_chunks = cnn_len // window_aftercnn
remainder = cnn_len % window_aftercnn
cu_chunk_lens.extend([window_aftercnn] * num_full_chunks)
+29 -25
View File
@@ -115,6 +115,7 @@ from vllm.utils.math_utils import round_up
from vllm.utils.torch_utils import PIN_MEMORY
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers
from ...utils.gpu_sync_debug import gpu_sync_allowed
from ...utils.torch_utils import async_tensor_h2d
from .interfaces import (
MultiModalEmbeddings,
@@ -2368,19 +2369,17 @@ class Qwen3VLForConditionalGeneration(
spatial_merge_size=self.visual.spatial_merge_size,
q=self.video_pruning_rate,
)
# Apply retention mask.
emb = emb[retention_mask]
# Calculate the actual number of retained tokens per frame.
num_frames, rows, cols = (
t,
h // merge_size,
w // merge_size,
)
retention_mask_thw = retention_mask.reshape(num_frames, rows, cols)
num_tokens_per_frame = (
retention_mask_thw.sum(dim=(1, 2)).long().tolist()
)
with gpu_sync_allowed():
# Apply retention mask.
emb = emb[retention_mask]
# Calculate the actual number of retained tokens per frame.
num_frames, rows, cols = t, h // merge_size, w // merge_size
retention_mask_thw = retention_mask.reshape(num_frames, rows, cols)
num_tokens_per_frame = (
retention_mask_thw.sum(dim=(1, 2)).long().tolist()
)
else:
feature_size = emb.shape[0] // num_frames
num_tokens_per_frame = [feature_size] * num_frames
@@ -2548,10 +2547,14 @@ class Qwen3VLForConditionalGeneration(
.permute(1, 0)
)
full_is_video_embed = unpruned_token_ids_tensor == embed_token_id
expanded_positions[is_video_embed, :3] = original_mrope[full_is_video_embed][
retention_mask
]
expanded_positions[~is_video_embed, :3] = original_mrope[~full_is_video_embed]
with gpu_sync_allowed():
expanded_positions[is_video_embed, :3] = original_mrope[
full_is_video_embed
][retention_mask]
expanded_positions[~is_video_embed, :3] = original_mrope[
~full_is_video_embed
]
expanded_positions[..., 3] = is_vision_start
expanded_positions[..., 4] = is_video_embed
@@ -2814,15 +2817,16 @@ class Qwen3VLForConditionalGeneration(
torch.empty(5, 0, device=device, dtype=torch.long)
)
positions, mrope_positions_delta = recompute_mrope_positions(
input_ids_t,
mm_embeddings_pos,
mrope_positions,
num_computed_tokens,
vision_start_token_id,
image_token_id,
video_token_id,
)
with gpu_sync_allowed():
positions, mrope_positions_delta = recompute_mrope_positions(
input_ids_t,
mm_embeddings_pos,
mrope_positions,
num_computed_tokens,
vision_start_token_id,
image_token_id,
video_token_id,
)
return mm_embeddings_out, positions, mrope_positions_delta
+20 -15
View File
@@ -54,6 +54,7 @@ from vllm.multimodal.processing import (
TimingContext,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsQuant
@@ -998,27 +999,31 @@ class SiglipEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):
position_diffs = position_ids[1:] - position_ids[:-1]
boundary_mask = position_diffs <= 0
boundary_indices = torch.cat(
[
torch.tensor([0], device=features.device),
torch.where(boundary_mask)[0] + 1,
torch.tensor([len(features)], device=features.device),
]
)
with gpu_sync_allowed():
boundary_mid_cpu = torch.where(boundary_mask.cpu())[0] + 1
zero = torch.zeros(1, dtype=boundary_mid_cpu.dtype)
end = torch.full((1,), len(features), dtype=boundary_mid_cpu.dtype)
boundary_indices_cpu = torch.cat([zero, boundary_mid_cpu, end])
# For each sequence [start, end), position i flips to: start + end - 1 - i
lengths = boundary_indices[1:] - boundary_indices[:-1]
starts = boundary_indices[:-1]
ends = boundary_indices[1:]
lengths_cpu = boundary_indices_cpu[1:] - boundary_indices_cpu[:-1]
starts_cpu = boundary_indices_cpu[:-1]
ends_cpu = boundary_indices_cpu[1:]
# Assign sequence ID to each element
sequence_ids = torch.arange(
len(lengths), device=features.device
).repeat_interleave(lengths)
sequence_ids_cpu = torch.arange(
len(lengths_cpu), dtype=boundary_mid_cpu.dtype
).repeat_interleave(lengths_cpu)
# Calculate flipped indices for all positions at once
current_positions = torch.arange(len(features), device=features.device)
flip_indices = starts[sequence_ids] + ends[sequence_ids] - 1 - current_positions
current_positions_cpu = torch.arange(
len(features), dtype=boundary_mid_cpu.dtype
)
flip_indices_cpu = (
starts_cpu[sequence_ids_cpu] + ends_cpu[sequence_ids_cpu]
) - (1 + current_positions_cpu)
flip_indices = flip_indices_cpu.to(features.device, non_blocking=True)
return features[flip_indices]
+4 -1
View File
@@ -61,6 +61,7 @@ from vllm.multimodal.processing import (
TimingContext,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from .interfaces import IsAttentionFree, MultiModalEmbeddings, SupportsMultiModal
from .interfaces_base import attn_type
@@ -280,7 +281,9 @@ class Terratorch(nn.Module, IsAttentionFree, SupportsMultiModal):
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
):
model_output = self.inference_runner.forward(**kwargs)
# terratorch's forward has internal GPU syncs.
with gpu_sync_allowed():
model_output = self.inference_runner.forward(**kwargs)
return model_output.output
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
@@ -54,6 +54,7 @@ from vllm.multimodal.processing import (
TimingContext,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
if TYPE_CHECKING:
from transformers import BatchFeature, PreTrainedModel
@@ -726,12 +727,17 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
kwargs.pop("token_type_ids", None)
kwargs.pop("mm_token_type_ids", None)
audio_output = self.model.get_audio_features(
input_features, return_dict=True, **kwargs
)
# HuggingFace's `get_audio_features` implementations branch on
# per-sample feature lengths internally.
with gpu_sync_allowed():
audio_output = self.model.get_audio_features(
input_features, return_dict=True, **kwargs
)
audio_embeddings = audio_output.pooler_output
split_sizes = num_audio_tokens.flatten().tolist()
# Per-audio token counts are needed as Python ints to split.
with gpu_sync_allowed():
split_sizes = num_audio_tokens.flatten().tolist()
return self._split_embeddings(audio_embeddings, split_sizes)
def _process_image_input(self, **kwargs) -> list[torch.Tensor] | None:
@@ -749,7 +755,12 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
num_image_patches = kwargs.pop("num_image_patches")
vision_embeddings = self.model.get_image_features(pixel_values, **kwargs)
# The underlying HuggingFace `get_image_features` implementations
# contain model-internal syncs (e.g. Idefics3 filters all-zero
# padding images via boolean-mask indexing, LlavaOnevision
# branches on per-sample batch counts).
with gpu_sync_allowed():
vision_embeddings = self.model.get_image_features(pixel_values, **kwargs)
# Transformers `v5`, `self.get_image_features` returns a tuple
# containing the features and optionally attentions/hidden_states
+13 -8
View File
@@ -43,6 +43,7 @@ from vllm.multimodal.processing import (
from vllm.renderers import TokenizeParams
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.ultravox import UltravoxConfig
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from .interfaces import (
@@ -841,15 +842,19 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
embeddings.shape[0], -1
)
mask = indices < audio_token_len[:, None]
# Apply mask and flatten
flattened_embeddings = embeddings[mask]
# Return one tensor per input audio
embed_lens = [
chunk_lens.sum().item()
for chunk_lens in audio_token_len.split(audio_input["num_chunks"].tolist())
]
return flattened_embeddings.split(embed_lens)
with gpu_sync_allowed():
# Apply mask and flatten
flattened_embeddings = embeddings[mask]
# Return one tensor per input audio
embed_lens = [
chunk_lens.sum().item()
for chunk_lens in audio_token_len.split(
audio_input["num_chunks"].tolist()
)
]
return flattened_embeddings.split(embed_lens)
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
audio_input = self._parse_and_validate_audio_input(**kwargs)