[Core] Check for GPU<->CPU syncs during CI (#43107)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
7b544ecb52
commit
615d4cfade
@@ -48,6 +48,7 @@ from vllm.model_executor.model_loader.weight_utils import (
|
||||
from vllm.model_executor.parameter import BasevLLMParameter
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
@@ -904,17 +905,21 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
# then chunk_stride = 2
|
||||
chunk_stride = mamba_block_size // chunk_size
|
||||
|
||||
# The per-sequence loop below uses these as Python scalars.
|
||||
# TODO avoid sync here?
|
||||
with gpu_sync_allowed():
|
||||
block_idx_first_cpu = block_idx_first_scheduled_token_p.tolist()
|
||||
block_idx_last_cpu = block_idx_last_scheduled_token_p.tolist()
|
||||
num_computed_tokens_p_cpu = num_computed_tokens_p.tolist()
|
||||
last_chunk_indices_p_cpu = last_chunk_indices_p.tolist()
|
||||
|
||||
# Save state for sequences with more than just final state
|
||||
for seq_idx in range(num_prefills):
|
||||
# Block index for the first scheduled token
|
||||
block_idx_first_scheduled_token = block_idx_first_scheduled_token_p[
|
||||
seq_idx
|
||||
]
|
||||
block_idx_first_scheduled_token = block_idx_first_cpu[seq_idx]
|
||||
|
||||
# Block index for the last scheduled token
|
||||
block_idx_last_scheduled_token = block_idx_last_scheduled_token_p[
|
||||
seq_idx
|
||||
]
|
||||
block_idx_last_scheduled_token = block_idx_last_cpu[seq_idx]
|
||||
|
||||
# Number of blocks that need to be written
|
||||
n_blocks_to_fill = (
|
||||
@@ -935,7 +940,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
if seq_idx == 0:
|
||||
first_chunk = 0
|
||||
else:
|
||||
first_chunk = 1 + last_chunk_indices_p[seq_idx - 1]
|
||||
first_chunk = 1 + last_chunk_indices_p_cpu[seq_idx - 1]
|
||||
|
||||
# First chunk that is aligned on the mamba block boundary
|
||||
first_aligned_chunk = first_chunk + chunk_stride - 1
|
||||
@@ -943,7 +948,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
# Calculate the number of computed tokens that were not
|
||||
# already cached
|
||||
num_unaligned_computed_tokens = (
|
||||
num_computed_tokens_p[seq_idx] % mamba_block_size
|
||||
num_computed_tokens_p_cpu[seq_idx] % mamba_block_size
|
||||
)
|
||||
|
||||
if num_unaligned_computed_tokens > 0:
|
||||
|
||||
@@ -54,6 +54,7 @@ from vllm.multimodal.processing import (
|
||||
PromptUpdateDetails,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
from .interfaces import (
|
||||
@@ -306,7 +307,10 @@ def _flatten_valid_audio_embeddings(
|
||||
< output_lengths[:, None]
|
||||
)
|
||||
|
||||
return audio_embeddings[valid_mask], output_lengths
|
||||
# Boolean-mask indexing has a data-dependent output shape, so the count
|
||||
# has to come back to the host.
|
||||
with gpu_sync_allowed():
|
||||
return audio_embeddings[valid_mask], output_lengths
|
||||
|
||||
|
||||
def _count_audio_tokens_from_mask(
|
||||
@@ -630,10 +634,10 @@ class AudioFlamingo3ForConditionalGeneration(
|
||||
audio_features,
|
||||
feature_attention_mask,
|
||||
)
|
||||
chunk_embeddings = torch.split(
|
||||
masked_audio_features,
|
||||
audio_output_lengths.tolist(),
|
||||
)
|
||||
# `split` needs Python int sizes.
|
||||
with gpu_sync_allowed():
|
||||
split_sizes = audio_output_lengths.tolist()
|
||||
chunk_embeddings = torch.split(masked_audio_features, split_sizes)
|
||||
|
||||
grouped_embeddings = []
|
||||
current_idx = 0
|
||||
|
||||
@@ -56,6 +56,7 @@ from vllm.transformers_utils.processors.cohere_asr import (
|
||||
CohereASRProcessor,
|
||||
)
|
||||
from vllm.utils.collection_utils import is_list_of
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionType,
|
||||
)
|
||||
@@ -1770,10 +1771,9 @@ class CohereASRModel(nn.Module):
|
||||
out = self.encoder_decoder_proj(out)
|
||||
|
||||
# Convert padded tensor to packed
|
||||
outs = []
|
||||
for i, feat in enumerate(out):
|
||||
feat_len = encoder_output_length[i]
|
||||
outs.append(feat[:feat_len, :])
|
||||
with gpu_sync_allowed():
|
||||
lengths = encoder_output_length.tolist()
|
||||
outs = [feat[:length, :] for feat, length in zip(out, lengths)]
|
||||
|
||||
return outs
|
||||
else:
|
||||
|
||||
@@ -71,6 +71,7 @@ from vllm.multimodal.processing import (
|
||||
PromptUpdate,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
@@ -788,7 +789,10 @@ class VariableResolutionResamplerModel(nn.Module):
|
||||
return x
|
||||
|
||||
def fwd_placeholder(x, grid_thw, to_tensor=False):
|
||||
grid_thw_cpu = grid_thw.cpu().numpy()
|
||||
# The per-image grids drive the Python-level offset arithmetic
|
||||
# below; the same read is wrapped on the cudagraph path.
|
||||
with gpu_sync_allowed():
|
||||
grid_thw_cpu = grid_thw.cpu().numpy()
|
||||
grid_t, grid_hw = grid_thw_cpu[:, 0], grid_thw_cpu[:, 1:]
|
||||
grid_hw_after_conv = grid_hw.prod(-1) // (self.spatial_conv_size**2)
|
||||
|
||||
@@ -1697,8 +1701,11 @@ class Ernie4_5_VLMoeForConditionalGeneration(
|
||||
EncoderCudaGraphReplayBuffers,
|
||||
)
|
||||
|
||||
# The per-image grids are needed as Python ints to size the buffers.
|
||||
with gpu_sync_allowed():
|
||||
grid_thw_list = mm_kwargs["image_grid_thw"].tolist()
|
||||
metadata = self.vision_model.prepare_encoder_metadata(
|
||||
mm_kwargs["image_grid_thw"].tolist(), max_batch_size=max_batch_size
|
||||
grid_thw_list, max_batch_size=max_batch_size
|
||||
)
|
||||
values = metadata | {
|
||||
"pixel_values": mm_kwargs["pixel_values"],
|
||||
@@ -1738,7 +1745,10 @@ class Ernie4_5_VLMoeForConditionalGeneration(
|
||||
# Ernie only uses the single "default" encoder path.
|
||||
output = outputs["default"]
|
||||
grid_thw = batch_mm_kwargs["image_grid_thw"].to(output.device)
|
||||
num_valid = int((grid_thw[:, 0] * grid_thw[:, 1] * grid_thw[:, 2]).sum())
|
||||
# The valid token count slices the graph output for the eager
|
||||
# resampler call, so it has to come back to the host.
|
||||
with gpu_sync_allowed():
|
||||
num_valid = int((grid_thw[:, 0] * grid_thw[:, 1] * grid_thw[:, 2]).sum())
|
||||
image_embeds = self.resampler_model(output[:num_valid], grid_thw)
|
||||
scatter_output_slices(image_embeds, indices, per_item_out_tokens, dest, clone)
|
||||
|
||||
|
||||
@@ -69,6 +69,7 @@ from vllm.multimodal.processing.processor import (
|
||||
PromptUpdateDetails,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
@@ -1365,11 +1366,14 @@ class Gemma4ForConditionalGeneration(
|
||||
pp_tensor,
|
||||
pad_tensor,
|
||||
).to(self.model_dtype)
|
||||
encoder_outputs = vt.encoder(
|
||||
inputs_embeds=inputs_embeds,
|
||||
attention_mask=~pad_tensor,
|
||||
pixel_position_ids=pp_tensor,
|
||||
)
|
||||
# HuggingFace's mask builder probes `padding_mask.all()` to
|
||||
# decide whether the mask can be skipped, which syncs.
|
||||
with gpu_sync_allowed():
|
||||
encoder_outputs = vt.encoder(
|
||||
inputs_embeds=inputs_embeds,
|
||||
attention_mask=~pad_tensor,
|
||||
pixel_position_ids=pp_tensor,
|
||||
)
|
||||
hidden_states = encoder_outputs.last_hidden_state
|
||||
|
||||
for i, (orig_idx, _, _) in enumerate(chunk_items):
|
||||
@@ -1387,13 +1391,17 @@ class Gemma4ForConditionalGeneration(
|
||||
single_pos_ids = pool_position_ids[orig_idx].unsqueeze(0)
|
||||
padding_positions = (single_pos_ids == -1).all(dim=-1)
|
||||
|
||||
pooled_states, valid_mask = vt.pooler(
|
||||
hidden_states=single_hidden,
|
||||
pixel_position_ids=single_pos_ids,
|
||||
padding_positions=padding_positions,
|
||||
output_length=output_length,
|
||||
)
|
||||
valid_states = pooled_states[valid_mask]
|
||||
# The pooler goes through HuggingFace's mask builder, which probes
|
||||
# `padding_mask.all()`, and the mask indexing below needs the
|
||||
# selected count on the host.
|
||||
with gpu_sync_allowed():
|
||||
pooled_states, valid_mask = vt.pooler(
|
||||
hidden_states=single_hidden,
|
||||
pixel_position_ids=single_pos_ids,
|
||||
padding_positions=padding_positions,
|
||||
output_length=output_length,
|
||||
)
|
||||
valid_states = pooled_states[valid_mask]
|
||||
|
||||
if getattr(vt.config, "standardize", False):
|
||||
valid_states = (valid_states - vt.std_bias) * vt.std_scale
|
||||
@@ -1444,7 +1452,9 @@ class Gemma4ForConditionalGeneration(
|
||||
pooling_k2 = vision_cfg.pooling_kernel_size**2
|
||||
|
||||
if isinstance(frame_counts, torch.Tensor):
|
||||
fc_list = frame_counts.tolist()
|
||||
# Per-video frame counts drive the Python-level batching below.
|
||||
with gpu_sync_allowed():
|
||||
fc_list = frame_counts.tolist()
|
||||
else:
|
||||
fc_list = list(frame_counts)
|
||||
|
||||
@@ -1474,11 +1484,13 @@ class Gemma4ForConditionalGeneration(
|
||||
pp_chunk,
|
||||
pad_chunk,
|
||||
).to(self.model_dtype)
|
||||
encoder_outputs = vt.encoder(
|
||||
inputs_embeds=inputs_embeds,
|
||||
attention_mask=~pad_chunk,
|
||||
pixel_position_ids=pp_chunk,
|
||||
)
|
||||
# HuggingFace's mask builder probes `padding_mask.all()`.
|
||||
with gpu_sync_allowed():
|
||||
encoder_outputs = vt.encoder(
|
||||
inputs_embeds=inputs_embeds,
|
||||
attention_mask=~pad_chunk,
|
||||
pixel_position_ids=pp_chunk,
|
||||
)
|
||||
last_hidden_states_list.append(encoder_outputs.last_hidden_state)
|
||||
|
||||
last_hidden_states = torch.cat(last_hidden_states_list, dim=0)
|
||||
@@ -1493,13 +1505,15 @@ class Gemma4ForConditionalGeneration(
|
||||
single_pos_ids = pixel_position_ids[i].unsqueeze(0)
|
||||
single_pad_pos = padding_positions[i].unsqueeze(0)
|
||||
|
||||
pooled_states, valid_mask = vt.pooler(
|
||||
hidden_states=single_hidden,
|
||||
pixel_position_ids=single_pos_ids,
|
||||
padding_positions=single_pad_pos,
|
||||
output_length=output_length,
|
||||
)
|
||||
valid_states = pooled_states[valid_mask]
|
||||
# As above, plus mask indexing that needs the count on the host.
|
||||
with gpu_sync_allowed():
|
||||
pooled_states, valid_mask = vt.pooler(
|
||||
hidden_states=single_hidden,
|
||||
pixel_position_ids=single_pos_ids,
|
||||
padding_positions=single_pad_pos,
|
||||
output_length=output_length,
|
||||
)
|
||||
valid_states = pooled_states[valid_mask]
|
||||
|
||||
if getattr(vt.config, "standardize", False):
|
||||
valid_states = (valid_states - vt.std_bias) * vt.std_scale
|
||||
@@ -1553,9 +1567,11 @@ class Gemma4ForConditionalGeneration(
|
||||
|
||||
# Strip padding per-batch element: only keep valid (non-padding)
|
||||
# tokens.
|
||||
# Boolean-mask indexing needs the selected count on the host.
|
||||
per_audio = []
|
||||
for enc, mask in zip(audio_features, audio_mask, strict=True):
|
||||
per_audio.append(enc[mask]) # [num_real, hidden_size]
|
||||
with gpu_sync_allowed():
|
||||
for enc, mask in zip(audio_features, audio_mask, strict=True):
|
||||
per_audio.append(enc[mask]) # [num_real, hidden_size]
|
||||
|
||||
return per_audio
|
||||
|
||||
|
||||
@@ -60,6 +60,7 @@ from vllm.multimodal.processing import (
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers import cached_tokenizer_from_config
|
||||
from vllm.transformers_utils.processor import cached_processor_from_config
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
|
||||
@@ -788,8 +789,9 @@ class GraniteSpeechForConditionalGeneration(
|
||||
encoder_embeds = self.encoder(audio_input["input_features"])
|
||||
# [bsz, <max feature size>, 4096]
|
||||
projected_embeds = self.projector(encoder_embeds)
|
||||
# Apply mask on variable length audio features
|
||||
masked_embeds = projected_embeds[audio_input["input_features_mask"]]
|
||||
# Apply mask on variable length audio features.
|
||||
with gpu_sync_allowed():
|
||||
masked_embeds = projected_embeds[audio_input["input_features_mask"]]
|
||||
# Split variable length features into a tuple
|
||||
return torch.split(masked_embeds, audio_input["audio_embed_sizes"])
|
||||
|
||||
|
||||
@@ -39,6 +39,7 @@ from vllm.model_executor.layers.linear import (
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
|
||||
from .utils import AutoWeightsLoader, WeightsMapper
|
||||
from .vision import is_vit_use_data_parallel, run_dp_sharded_vision_model
|
||||
@@ -96,10 +97,14 @@ class Idefics2VisionEmbeddings(nn.Module):
|
||||
size=(batch_size, max_nb_patches_h * max_nb_patches_w), fill_value=0
|
||||
)
|
||||
|
||||
for batch_idx, p_attn_mask in enumerate(patch_attention_mask):
|
||||
if tgt_sizes is not None:
|
||||
nb_patches_h = tgt_sizes[batch_idx][0]
|
||||
nb_patches_w = tgt_sizes[batch_idx][1]
|
||||
with gpu_sync_allowed():
|
||||
patch_attention_mask_cpu = patch_attention_mask.cpu()
|
||||
tgt_sizes_cpu = tgt_sizes.cpu() if tgt_sizes is not None else None
|
||||
|
||||
for batch_idx, p_attn_mask in enumerate(patch_attention_mask_cpu):
|
||||
if tgt_sizes_cpu is not None:
|
||||
nb_patches_h = tgt_sizes_cpu[batch_idx][0]
|
||||
nb_patches_w = tgt_sizes_cpu[batch_idx][1]
|
||||
else:
|
||||
nb_patches_h = p_attn_mask[:, 0].sum()
|
||||
nb_patches_w = p_attn_mask[0].sum()
|
||||
@@ -114,8 +119,10 @@ class Idefics2VisionEmbeddings(nn.Module):
|
||||
pos_ids = (
|
||||
bucket_coords_h[:, None] * self.num_patches_per_side + bucket_coords_w
|
||||
).flatten()
|
||||
position_ids[batch_idx][p_attn_mask.view(-1).cpu()] = pos_ids
|
||||
position_ids = position_ids.to(self.position_embedding.weight.device)
|
||||
position_ids[batch_idx][p_attn_mask.view(-1)] = pos_ids
|
||||
position_ids = position_ids.to(
|
||||
self.position_embedding.weight.device, non_blocking=True
|
||||
)
|
||||
embeddings += self.position_embedding(position_ids)
|
||||
return embeddings
|
||||
|
||||
|
||||
@@ -51,6 +51,7 @@ from vllm.multimodal.processing import (
|
||||
PromptUpdateDetails,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
from .idefics2_vision_model import (
|
||||
@@ -443,11 +444,12 @@ class Idefics3Model(nn.Module):
|
||||
real_images_inds = (pixel_values == 0.0).sum(
|
||||
dim=(-1, -2, -3)
|
||||
) != nb_values_per_image
|
||||
pixel_values = pixel_values[real_images_inds].contiguous()
|
||||
with gpu_sync_allowed():
|
||||
pixel_values = pixel_values[real_images_inds].contiguous()
|
||||
|
||||
# Handle the vision attention mask
|
||||
# Remove padding images from the mask
|
||||
pixel_attention_mask = pixel_attention_mask[real_images_inds].contiguous()
|
||||
# Handle the vision attention mask
|
||||
# Remove padding images from the mask
|
||||
pixel_attention_mask = pixel_attention_mask[real_images_inds].contiguous()
|
||||
|
||||
patch_size = self.config.vision_config.patch_size
|
||||
patches_subgrid = pixel_attention_mask.unfold(
|
||||
|
||||
@@ -16,6 +16,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.transformers_utils.repo_utils import get_hf_file_bytes
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.v1.pool.metadata import PoolingMetadata
|
||||
|
||||
from ..layers.pooler import DispatchPooler
|
||||
@@ -110,19 +111,24 @@ class JinaForRankingPool(StepPool):
|
||||
prompt_token_ids = pooling_metadata.get_prompt_token_ids()
|
||||
|
||||
embeds_list = list[torch.Tensor | None]()
|
||||
for data, token_ids in zip(pooled_data_lst, prompt_token_ids):
|
||||
# for unfinished chunked prefill
|
||||
if data is None:
|
||||
embeds_list.append(None)
|
||||
else:
|
||||
docs_indexes = torch.where(torch.eq(token_ids, self.doc_token_id))[0]
|
||||
query_indexes = torch.where(torch.eq(token_ids, self.query_token_id))[0]
|
||||
# `torch.where` resolves the match count on the host.
|
||||
with gpu_sync_allowed():
|
||||
for data, token_ids in zip(pooled_data_lst, prompt_token_ids):
|
||||
# for unfinished chunked prefill
|
||||
if data is None:
|
||||
embeds_list.append(None)
|
||||
else:
|
||||
doc_match = torch.eq(token_ids, self.doc_token_id)
|
||||
docs_indexes = torch.where(doc_match)[0]
|
||||
query_indexes = torch.where(
|
||||
torch.eq(token_ids, self.query_token_id)
|
||||
)[0]
|
||||
|
||||
# The JinaForRanking model concatenates docs first, then query.
|
||||
# Let's stay consistent with this novel design.
|
||||
indexes = torch.cat([docs_indexes, query_indexes])
|
||||
embeds = self.projector(data[indexes])
|
||||
embeds_list.append(embeds)
|
||||
# The JinaForRanking model concatenates docs first, then query.
|
||||
# Let's stay consistent with this novel design.
|
||||
indexes = torch.cat([docs_indexes, query_indexes])
|
||||
embeds = self.projector(data[indexes])
|
||||
embeds_list.append(embeds)
|
||||
|
||||
return embeds_list
|
||||
|
||||
|
||||
@@ -98,6 +98,7 @@ from vllm.transformers_utils.processor import (
|
||||
)
|
||||
from vllm.transformers_utils.utils import convert_model_repo_to_path
|
||||
from vllm.utils.collection_utils import flatten_2d_lists
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
|
||||
@@ -1548,7 +1549,8 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
|
||||
all_pixel_values[i, ..., :L_item] = pixel_values_item
|
||||
|
||||
num_patches = tgt_sizes.prod(-1)
|
||||
max_patches = num_patches.max().item()
|
||||
with gpu_sync_allowed():
|
||||
max_patches = num_patches.max().item()
|
||||
assert isinstance(max_patches, int)
|
||||
|
||||
patch_attn_mask = torch.zeros((B, max_patches), dtype=torch.bool, device=device)
|
||||
@@ -1640,7 +1642,9 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
|
||||
all_pixel_values[i, ..., :L_item] = pixel_values_item
|
||||
|
||||
num_patches = tgt_sizes.prod(-1)
|
||||
max_patches = num_patches.max().item()
|
||||
# Needed as a Python int to size the mask below.
|
||||
with gpu_sync_allowed():
|
||||
max_patches = num_patches.max().item()
|
||||
assert isinstance(max_patches, int)
|
||||
|
||||
patch_attn_mask = torch.zeros((B, max_patches), dtype=torch.bool, device=device)
|
||||
@@ -1737,7 +1741,8 @@ class MiniCPMV4_0(MiniCPMVBaseModel, SupportsLoRA):
|
||||
all_pixel_values[i, ..., :L_item] = pixel_values_item
|
||||
|
||||
num_patches = tgt_sizes.prod(-1)
|
||||
max_patches = num_patches.max().item()
|
||||
with gpu_sync_allowed():
|
||||
max_patches = num_patches.max().item()
|
||||
assert isinstance(max_patches, int)
|
||||
|
||||
patch_attn_mask = torch.zeros((B, max_patches), dtype=torch.bool, device=device)
|
||||
@@ -1839,7 +1844,8 @@ class MiniCPMV4_5(MiniCPMVBaseModel, SupportsLoRA):
|
||||
all_pixel_values[i, ..., :L_item] = pixel_values_item
|
||||
|
||||
num_patches = tgt_sizes.prod(-1)
|
||||
max_patches = num_patches.max().item()
|
||||
with gpu_sync_allowed():
|
||||
max_patches = num_patches.max().item()
|
||||
assert isinstance(max_patches, int)
|
||||
|
||||
patch_attn_mask = torch.zeros((B, max_patches), dtype=torch.bool, device=device)
|
||||
|
||||
@@ -66,6 +66,7 @@ from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.model_executor.models.vision import is_vit_use_data_parallel
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.configs.moonvit import MoonViTConfig
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
|
||||
@@ -811,9 +812,14 @@ class MoonVitPretrainedModel(PreTrainedModel):
|
||||
merge_kernel_size=self.merge_kernel_size,
|
||||
)
|
||||
|
||||
hidden_states = self.patch_embed(pixel_values, grid_hw)
|
||||
hidden_states = self.encoder(hidden_states, grid_hw)
|
||||
hidden_states = patch_merger(
|
||||
hidden_states, grid_hw, merge_kernel_size=self.merge_kernel_size
|
||||
)
|
||||
# Legacy path: patch_embed, encoder and patch_merger each iterate the
|
||||
# per-image grids in Python, so all three read `grid_hw` back to the
|
||||
# host. The `encoder_metadata` path above precomputes that outside the
|
||||
# graph and does not sync.
|
||||
with gpu_sync_allowed():
|
||||
hidden_states = self.patch_embed(pixel_values, grid_hw)
|
||||
hidden_states = self.encoder(hidden_states, grid_hw)
|
||||
hidden_states = patch_merger(
|
||||
hidden_states, grid_hw, merge_kernel_size=self.merge_kernel_size
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
@@ -55,6 +55,7 @@ from vllm.multimodal.processing import (
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
from .interfaces import (
|
||||
@@ -1667,15 +1668,19 @@ class MossAudioModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
|
||||
"""
|
||||
audio_data = audio_input["audio_data"]
|
||||
audio_data_seqlens = audio_input["audio_data_seqlens"]
|
||||
last_hidden_state, deepstack = self.audio_encoder(
|
||||
audio_data.to(self.audio_encoder.dtype),
|
||||
feature_lens=audio_data_seqlens,
|
||||
output_deepstack_hidden_states=len(self.deepstack_audio_merger_list) > 0,
|
||||
)
|
||||
audio_embeds = self.audio_adapter(last_hidden_state)
|
||||
audio_lengths = MossAudioEncoder._compute_downsampled_length(
|
||||
audio_data_seqlens.to(device=audio_embeds.device, dtype=torch.long)
|
||||
).tolist()
|
||||
# The encoder chunks the input by per-audio feature lengths, which
|
||||
# needs Python ints for `split`/`pad_sequence`.
|
||||
want_deepstack = len(self.deepstack_audio_merger_list) > 0
|
||||
with gpu_sync_allowed():
|
||||
last_hidden_state, deepstack = self.audio_encoder(
|
||||
audio_data.to(self.audio_encoder.dtype),
|
||||
feature_lens=audio_data_seqlens,
|
||||
output_deepstack_hidden_states=want_deepstack,
|
||||
)
|
||||
audio_embeds = self.audio_adapter(last_hidden_state)
|
||||
audio_lengths = MossAudioEncoder._compute_downsampled_length(
|
||||
audio_data_seqlens.to(device=audio_embeds.device, dtype=torch.long)
|
||||
).tolist()
|
||||
main_embeddings = tuple(audio_embeds.squeeze(0).split(audio_lengths, dim=0))
|
||||
|
||||
deepstack_embeddings: list[tuple[torch.Tensor, ...]] = []
|
||||
|
||||
@@ -49,6 +49,7 @@ from vllm.multimodal.processing.processor import (
|
||||
ResolvedPromptUpdate,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
from .idefics2_vision_model import Idefics2VisionTransformer
|
||||
@@ -396,12 +397,23 @@ class Phi4MMImageEncoder(nn.Module):
|
||||
w * base_feat_width // base_feat_height_reduction,
|
||||
)
|
||||
)
|
||||
useful_height = int(reshaped_image_attention_mask[0, :, 0].sum().item())
|
||||
useful_width = int(reshaped_image_attention_mask[0, 0, :].sum().item())
|
||||
# The mask stays on device for the encoder above, so these
|
||||
# per-image counts have to come back to the host to drive the
|
||||
# slicing and the Python-level length arithmetic.
|
||||
with gpu_sync_allowed():
|
||||
useful_height = int(
|
||||
reshaped_image_attention_mask[0, :, 0].sum().item()
|
||||
)
|
||||
useful_width = int(
|
||||
reshaped_image_attention_mask[0, 0, :].sum().item()
|
||||
)
|
||||
mask_token_count = int(
|
||||
image_attention_mask[_bs, : B_ + 1, 0::2, 0::2].sum().item()
|
||||
)
|
||||
sub_img = sub_img[:, :useful_height, :useful_width]
|
||||
temp_sub_GN = self.sub_GN.repeat(1, useful_height, 1, 1)
|
||||
temp_len = (
|
||||
int(image_attention_mask[_bs, : B_ + 1, 0::2, 0::2].sum().item())
|
||||
mask_token_count
|
||||
+ (useful_height + 1)
|
||||
+ base_feat_height // base_feat_height_reduction
|
||||
)
|
||||
|
||||
@@ -89,6 +89,7 @@ from vllm.multimodal.processing.processor import (
|
||||
PromptUpdateDetails,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
from .interfaces import (
|
||||
@@ -1036,12 +1037,14 @@ class Qwen2_5OmniConditionalGenerationMixin:
|
||||
self.audio_tower._get_feat_extract_output_lengths(audio_feature_lengths)
|
||||
)
|
||||
|
||||
audio_outputs = self.audio_tower(
|
||||
input_features.to(self.audio_tower.dtype),
|
||||
feature_lens=audio_feature_lengths,
|
||||
aftercnn_lens=audio_feat_lengths,
|
||||
)
|
||||
return audio_outputs.last_hidden_state.split(audio_output_lengths.tolist())
|
||||
with gpu_sync_allowed():
|
||||
audio_outputs = self.audio_tower(
|
||||
input_features.to(self.audio_tower.dtype),
|
||||
feature_lens=audio_feature_lengths,
|
||||
aftercnn_lens=audio_feat_lengths,
|
||||
)
|
||||
split_sizes = audio_output_lengths.tolist()
|
||||
return audio_outputs.last_hidden_state.split(split_sizes)
|
||||
|
||||
def _process_image_input(
|
||||
self, image_input: Qwen2_5_VLImageInputs
|
||||
@@ -1526,12 +1529,13 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
|
||||
video_token_id = self.config.video_token_index
|
||||
audio_token_id = self.config.audio_token_index
|
||||
|
||||
input_ids_cpu = input_ids.cpu()
|
||||
is_video = is_multimodal & (input_ids_cpu == video_token_id)
|
||||
is_audio = is_multimodal & (input_ids_cpu == audio_token_id)
|
||||
with gpu_sync_allowed():
|
||||
input_ids_cpu = input_ids.cpu()
|
||||
is_video = is_multimodal & (input_ids_cpu == video_token_id)
|
||||
is_audio = is_multimodal & (input_ids_cpu == audio_token_id)
|
||||
|
||||
num_video = is_video.sum().item()
|
||||
num_audio = is_audio.sum().item()
|
||||
num_video = is_video.sum().item()
|
||||
num_audio = is_audio.sum().item()
|
||||
|
||||
if check_interleaved_audio_video(is_video, is_audio, num_video, num_audio):
|
||||
inputs_embeds = self._embed_text_input_ids(
|
||||
|
||||
@@ -87,6 +87,7 @@ from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers
|
||||
|
||||
from ...utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from .interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
SupportsEagle,
|
||||
@@ -1569,8 +1570,9 @@ class Qwen2_5_VLForConditionalGeneration(
|
||||
video_second_per_grid=video_second_per_grid_t.item(),
|
||||
).to(emb.device, non_blocking=True)
|
||||
|
||||
emb = emb[retention_mask]
|
||||
positions = positions[retention_mask]
|
||||
with gpu_sync_allowed():
|
||||
emb = emb[retention_mask]
|
||||
positions = positions[retention_mask]
|
||||
emb = torch.cat([emb, positions], dim=1)
|
||||
video_embeds_out.append(emb)
|
||||
return tuple(video_embeds_out)
|
||||
@@ -1624,15 +1626,16 @@ class Qwen2_5_VLForConditionalGeneration(
|
||||
mm[:, -4:].permute(1, 0).long() for mm in multimodal_embeddings
|
||||
]
|
||||
|
||||
positions, mrope_positions_delta = recompute_mrope_positions(
|
||||
input_ids_t,
|
||||
mm_embeddings_pos,
|
||||
mrope_positions,
|
||||
num_computed_tokens,
|
||||
vision_start_token_id,
|
||||
image_token_id,
|
||||
video_token_id,
|
||||
)
|
||||
with gpu_sync_allowed():
|
||||
positions, mrope_positions_delta = recompute_mrope_positions(
|
||||
input_ids_t,
|
||||
mm_embeddings_pos,
|
||||
mrope_positions,
|
||||
num_computed_tokens,
|
||||
vision_start_token_id,
|
||||
image_token_id,
|
||||
video_token_id,
|
||||
)
|
||||
|
||||
return mm_embeddings_out, positions, mrope_positions_delta
|
||||
|
||||
|
||||
@@ -60,6 +60,7 @@ from vllm.multimodal.processing import (
|
||||
PromptUpdate,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
|
||||
@@ -444,12 +445,15 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal, Supports
|
||||
)
|
||||
< audio_output_lengths
|
||||
)
|
||||
masked_audio_features = audio_features[audio_features_mask].view(-1, embed_dim)
|
||||
with gpu_sync_allowed():
|
||||
masked_audio_features = audio_features[audio_features_mask].view(
|
||||
-1, embed_dim
|
||||
)
|
||||
|
||||
# Split to tuple of embeddings for individual audio input.
|
||||
return torch.split(
|
||||
masked_audio_features, audio_output_lengths.flatten().tolist()
|
||||
)
|
||||
# Split to tuple of embeddings for individual audio input.
|
||||
return torch.split(
|
||||
masked_audio_features, audio_output_lengths.flatten().tolist()
|
||||
)
|
||||
|
||||
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
|
||||
audio_input = self._parse_and_validate_audio_input(**kwargs)
|
||||
|
||||
@@ -92,6 +92,7 @@ from vllm.transformers_utils.processor import cached_processor_from_config
|
||||
from vllm.transformers_utils.processors.qwen3_asr import (
|
||||
Qwen3ASRProcessor,
|
||||
)
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
|
||||
logger = init_logger(__name__)
|
||||
_ASR_TEXT_TAG = "<asr_text>"
|
||||
@@ -467,7 +468,9 @@ class Qwen3ASRForConditionalGeneration(
|
||||
feature_lens=audio_feature_lengths,
|
||||
aftercnn_lens=audio_output_lengths,
|
||||
)
|
||||
return audio_features.split(audio_output_lengths.tolist())
|
||||
with gpu_sync_allowed():
|
||||
split_sizes = audio_output_lengths.tolist()
|
||||
return audio_features.split(split_sizes)
|
||||
|
||||
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings | None:
|
||||
mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)
|
||||
|
||||
@@ -77,6 +77,7 @@ from vllm.multimodal.processing.processor import (
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.processor import cached_processor_from_config
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
@@ -434,11 +435,14 @@ class Qwen3OmniMoeAudioEncoder(nn.Module):
|
||||
feature_lens: torch.Tensor,
|
||||
aftercnn_lens: torch.Tensor,
|
||||
):
|
||||
# Compute chunk information
|
||||
# Compute chunk information.
|
||||
chunk_num = torch.ceil(feature_lens / (self.n_window * 2)).long()
|
||||
|
||||
with gpu_sync_allowed():
|
||||
total_chunks = int(chunk_num.sum())
|
||||
|
||||
chunk_lengths = async_tensor_h2d(
|
||||
[self.n_window * 2] * chunk_num.sum(),
|
||||
[self.n_window * 2] * total_chunks,
|
||||
dtype=torch.long,
|
||||
device=feature_lens.device,
|
||||
)
|
||||
@@ -447,15 +451,18 @@ class Qwen3OmniMoeAudioEncoder(nn.Module):
|
||||
chunk_lengths[chunk_lengths == 0] = self.n_window * 2
|
||||
|
||||
# Split input features into chunks and pad
|
||||
chunk_list = input_features.T.split(chunk_lengths.tolist(), dim=0)
|
||||
with gpu_sync_allowed():
|
||||
chunk_lengths_list = chunk_lengths.tolist()
|
||||
chunk_list = input_features.T.split(chunk_lengths_list, dim=0)
|
||||
padded_feature = nn.utils.rnn.pad_sequence(
|
||||
chunk_list, batch_first=True
|
||||
).transpose(1, 2)
|
||||
|
||||
# Compute feature lengths after CNN
|
||||
feature_lens_after_cnn = self._get_cnn_output_lengths(chunk_lengths)
|
||||
# Vectorized mask creation: avoid creating many small tensors
|
||||
max_len_after_cnn = feature_lens_after_cnn.max().item()
|
||||
# Vectorized mask creation: avoid creating many small tensors.
|
||||
with gpu_sync_allowed():
|
||||
max_len_after_cnn = feature_lens_after_cnn.max().item()
|
||||
indices = torch.arange(max_len_after_cnn, device=padded_feature.device)
|
||||
padded_mask_after_cnn = indices.unsqueeze(0) < feature_lens_after_cnn.unsqueeze(
|
||||
1
|
||||
@@ -494,16 +501,17 @@ class Qwen3OmniMoeAudioEncoder(nn.Module):
|
||||
)
|
||||
padded_embed = padded_embed + positional_embedding
|
||||
|
||||
# Extract valid hidden states and compute cu_seqlens
|
||||
hidden_states = padded_embed[padded_mask_after_cnn]
|
||||
with gpu_sync_allowed():
|
||||
hidden_states = padded_embed[padded_mask_after_cnn]
|
||||
# Use tolist() for efficient batch conversion from tensor to Python.
|
||||
aftercnn_lens_list = aftercnn_lens.tolist()
|
||||
|
||||
# Compute cumulative sequence lengths for chunked attention
|
||||
cu_chunk_lens = [0]
|
||||
window_aftercnn = padded_mask_after_cnn.shape[-1] * (
|
||||
self.n_window_infer // (self.n_window * 2)
|
||||
)
|
||||
# Use tolist() for efficient batch conversion from tensor to Python
|
||||
for cnn_len in aftercnn_lens.tolist():
|
||||
for cnn_len in aftercnn_lens_list:
|
||||
num_full_chunks = cnn_len // window_aftercnn
|
||||
remainder = cnn_len % window_aftercnn
|
||||
cu_chunk_lens.extend([window_aftercnn] * num_full_chunks)
|
||||
|
||||
@@ -115,6 +115,7 @@ from vllm.utils.math_utils import round_up
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers
|
||||
|
||||
from ...utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from ...utils.torch_utils import async_tensor_h2d
|
||||
from .interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
@@ -2368,19 +2369,17 @@ class Qwen3VLForConditionalGeneration(
|
||||
spatial_merge_size=self.visual.spatial_merge_size,
|
||||
q=self.video_pruning_rate,
|
||||
)
|
||||
# Apply retention mask.
|
||||
emb = emb[retention_mask]
|
||||
|
||||
# Calculate the actual number of retained tokens per frame.
|
||||
num_frames, rows, cols = (
|
||||
t,
|
||||
h // merge_size,
|
||||
w // merge_size,
|
||||
)
|
||||
retention_mask_thw = retention_mask.reshape(num_frames, rows, cols)
|
||||
num_tokens_per_frame = (
|
||||
retention_mask_thw.sum(dim=(1, 2)).long().tolist()
|
||||
)
|
||||
with gpu_sync_allowed():
|
||||
# Apply retention mask.
|
||||
emb = emb[retention_mask]
|
||||
|
||||
# Calculate the actual number of retained tokens per frame.
|
||||
num_frames, rows, cols = t, h // merge_size, w // merge_size
|
||||
retention_mask_thw = retention_mask.reshape(num_frames, rows, cols)
|
||||
num_tokens_per_frame = (
|
||||
retention_mask_thw.sum(dim=(1, 2)).long().tolist()
|
||||
)
|
||||
else:
|
||||
feature_size = emb.shape[0] // num_frames
|
||||
num_tokens_per_frame = [feature_size] * num_frames
|
||||
@@ -2548,10 +2547,14 @@ class Qwen3VLForConditionalGeneration(
|
||||
.permute(1, 0)
|
||||
)
|
||||
full_is_video_embed = unpruned_token_ids_tensor == embed_token_id
|
||||
expanded_positions[is_video_embed, :3] = original_mrope[full_is_video_embed][
|
||||
retention_mask
|
||||
]
|
||||
expanded_positions[~is_video_embed, :3] = original_mrope[~full_is_video_embed]
|
||||
|
||||
with gpu_sync_allowed():
|
||||
expanded_positions[is_video_embed, :3] = original_mrope[
|
||||
full_is_video_embed
|
||||
][retention_mask]
|
||||
expanded_positions[~is_video_embed, :3] = original_mrope[
|
||||
~full_is_video_embed
|
||||
]
|
||||
expanded_positions[..., 3] = is_vision_start
|
||||
expanded_positions[..., 4] = is_video_embed
|
||||
|
||||
@@ -2814,15 +2817,16 @@ class Qwen3VLForConditionalGeneration(
|
||||
torch.empty(5, 0, device=device, dtype=torch.long)
|
||||
)
|
||||
|
||||
positions, mrope_positions_delta = recompute_mrope_positions(
|
||||
input_ids_t,
|
||||
mm_embeddings_pos,
|
||||
mrope_positions,
|
||||
num_computed_tokens,
|
||||
vision_start_token_id,
|
||||
image_token_id,
|
||||
video_token_id,
|
||||
)
|
||||
with gpu_sync_allowed():
|
||||
positions, mrope_positions_delta = recompute_mrope_positions(
|
||||
input_ids_t,
|
||||
mm_embeddings_pos,
|
||||
mrope_positions,
|
||||
num_computed_tokens,
|
||||
vision_start_token_id,
|
||||
image_token_id,
|
||||
video_token_id,
|
||||
)
|
||||
|
||||
return mm_embeddings_out, positions, mrope_positions_delta
|
||||
|
||||
|
||||
@@ -54,6 +54,7 @@ from vllm.multimodal.processing import (
|
||||
TimingContext,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsQuant
|
||||
@@ -998,27 +999,31 @@ class SiglipEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):
|
||||
position_diffs = position_ids[1:] - position_ids[:-1]
|
||||
boundary_mask = position_diffs <= 0
|
||||
|
||||
boundary_indices = torch.cat(
|
||||
[
|
||||
torch.tensor([0], device=features.device),
|
||||
torch.where(boundary_mask)[0] + 1,
|
||||
torch.tensor([len(features)], device=features.device),
|
||||
]
|
||||
)
|
||||
with gpu_sync_allowed():
|
||||
boundary_mid_cpu = torch.where(boundary_mask.cpu())[0] + 1
|
||||
|
||||
zero = torch.zeros(1, dtype=boundary_mid_cpu.dtype)
|
||||
end = torch.full((1,), len(features), dtype=boundary_mid_cpu.dtype)
|
||||
boundary_indices_cpu = torch.cat([zero, boundary_mid_cpu, end])
|
||||
|
||||
# For each sequence [start, end), position i flips to: start + end - 1 - i
|
||||
lengths = boundary_indices[1:] - boundary_indices[:-1]
|
||||
starts = boundary_indices[:-1]
|
||||
ends = boundary_indices[1:]
|
||||
lengths_cpu = boundary_indices_cpu[1:] - boundary_indices_cpu[:-1]
|
||||
starts_cpu = boundary_indices_cpu[:-1]
|
||||
ends_cpu = boundary_indices_cpu[1:]
|
||||
|
||||
# Assign sequence ID to each element
|
||||
sequence_ids = torch.arange(
|
||||
len(lengths), device=features.device
|
||||
).repeat_interleave(lengths)
|
||||
sequence_ids_cpu = torch.arange(
|
||||
len(lengths_cpu), dtype=boundary_mid_cpu.dtype
|
||||
).repeat_interleave(lengths_cpu)
|
||||
|
||||
# Calculate flipped indices for all positions at once
|
||||
current_positions = torch.arange(len(features), device=features.device)
|
||||
flip_indices = starts[sequence_ids] + ends[sequence_ids] - 1 - current_positions
|
||||
current_positions_cpu = torch.arange(
|
||||
len(features), dtype=boundary_mid_cpu.dtype
|
||||
)
|
||||
flip_indices_cpu = (
|
||||
starts_cpu[sequence_ids_cpu] + ends_cpu[sequence_ids_cpu]
|
||||
) - (1 + current_positions_cpu)
|
||||
flip_indices = flip_indices_cpu.to(features.device, non_blocking=True)
|
||||
|
||||
return features[flip_indices]
|
||||
|
||||
|
||||
@@ -61,6 +61,7 @@ from vllm.multimodal.processing import (
|
||||
TimingContext,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
|
||||
from .interfaces import IsAttentionFree, MultiModalEmbeddings, SupportsMultiModal
|
||||
from .interfaces_base import attn_type
|
||||
@@ -280,7 +281,9 @@ class Terratorch(nn.Module, IsAttentionFree, SupportsMultiModal):
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
**kwargs: object,
|
||||
):
|
||||
model_output = self.inference_runner.forward(**kwargs)
|
||||
# terratorch's forward has internal GPU syncs.
|
||||
with gpu_sync_allowed():
|
||||
model_output = self.inference_runner.forward(**kwargs)
|
||||
return model_output.output
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
|
||||
@@ -54,6 +54,7 @@ from vllm.multimodal.processing import (
|
||||
TimingContext,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from transformers import BatchFeature, PreTrainedModel
|
||||
@@ -726,12 +727,17 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
kwargs.pop("token_type_ids", None)
|
||||
kwargs.pop("mm_token_type_ids", None)
|
||||
|
||||
audio_output = self.model.get_audio_features(
|
||||
input_features, return_dict=True, **kwargs
|
||||
)
|
||||
# HuggingFace's `get_audio_features` implementations branch on
|
||||
# per-sample feature lengths internally.
|
||||
with gpu_sync_allowed():
|
||||
audio_output = self.model.get_audio_features(
|
||||
input_features, return_dict=True, **kwargs
|
||||
)
|
||||
audio_embeddings = audio_output.pooler_output
|
||||
|
||||
split_sizes = num_audio_tokens.flatten().tolist()
|
||||
# Per-audio token counts are needed as Python ints to split.
|
||||
with gpu_sync_allowed():
|
||||
split_sizes = num_audio_tokens.flatten().tolist()
|
||||
return self._split_embeddings(audio_embeddings, split_sizes)
|
||||
|
||||
def _process_image_input(self, **kwargs) -> list[torch.Tensor] | None:
|
||||
@@ -749,7 +755,12 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
|
||||
num_image_patches = kwargs.pop("num_image_patches")
|
||||
|
||||
vision_embeddings = self.model.get_image_features(pixel_values, **kwargs)
|
||||
# The underlying HuggingFace `get_image_features` implementations
|
||||
# contain model-internal syncs (e.g. Idefics3 filters all-zero
|
||||
# padding images via boolean-mask indexing, LlavaOnevision
|
||||
# branches on per-sample batch counts).
|
||||
with gpu_sync_allowed():
|
||||
vision_embeddings = self.model.get_image_features(pixel_values, **kwargs)
|
||||
|
||||
# Transformers `v5`, `self.get_image_features` returns a tuple
|
||||
# containing the features and optionally attentions/hidden_states
|
||||
|
||||
@@ -43,6 +43,7 @@ from vllm.multimodal.processing import (
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.ultravox import UltravoxConfig
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
from .interfaces import (
|
||||
@@ -841,15 +842,19 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
|
||||
embeddings.shape[0], -1
|
||||
)
|
||||
mask = indices < audio_token_len[:, None]
|
||||
# Apply mask and flatten
|
||||
flattened_embeddings = embeddings[mask]
|
||||
|
||||
# Return one tensor per input audio
|
||||
embed_lens = [
|
||||
chunk_lens.sum().item()
|
||||
for chunk_lens in audio_token_len.split(audio_input["num_chunks"].tolist())
|
||||
]
|
||||
return flattened_embeddings.split(embed_lens)
|
||||
with gpu_sync_allowed():
|
||||
# Apply mask and flatten
|
||||
flattened_embeddings = embeddings[mask]
|
||||
|
||||
# Return one tensor per input audio
|
||||
embed_lens = [
|
||||
chunk_lens.sum().item()
|
||||
for chunk_lens in audio_token_len.split(
|
||||
audio_input["num_chunks"].tolist()
|
||||
)
|
||||
]
|
||||
return flattened_embeddings.split(embed_lens)
|
||||
|
||||
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
|
||||
audio_input = self._parse_and_validate_audio_input(**kwargs)
|
||||
|
||||
Reference in New Issue
Block a user