[Perf] Avoid some more unnecessary GPU<->CPU syncs (#51458)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-08-08 17:13:26 -07:00
committed by GitHub
parent d608dfabfd
commit 9b0afeb4f6
13 changed files with 99 additions and 49 deletions
+32 -4
View File
@@ -55,6 +55,7 @@ from vllm.multimodal.processing import (
)
from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.utils.torch_utils import async_tensor_h2d
from .interfaces import (
MultiModalEmbeddings,
@@ -819,8 +820,20 @@ class ChameleonImageVocabularyMapping:
def convert_img2bpe(self, img_batch: torch.Tensor) -> torch.Tensor:
device = img_batch.device
img_tokens = self.img2bpe_mapping_tensor[img_batch.to("cpu")]
return img_tokens.to(device)
# Cache a per-device copy of the (small, static) mapping tensor so we
# can index entirely on `device` instead of forcing a D2H on
# `img_batch` and an H2D on the result.
cache = getattr(self, "_img2bpe_mapping_cache", None)
if cache is None:
cache = {}
self._img2bpe_mapping_cache = cache
mapping_on_device = cache.get(device)
if mapping_on_device is None:
mapping_on_device = async_tensor_h2d(
self.img2bpe_mapping_tensor, device=device
)
cache[device] = mapping_on_device
return mapping_on_device[img_batch]
class ChameleonModel(nn.Module):
@@ -1025,8 +1038,23 @@ class ChameleonForConditionalGeneration(
# Disallow image tokens which does not include special
# begin-image and end-image tokens
if logits is not None:
image_tokens = self.model.vocabulary_mapping.image_tokens
logits[:, image_tokens] = torch.finfo(logits.dtype).min
# Cache a per-device index tensor for the (static) image-token
# set, and use `index_fill_` instead of advanced-index assign
# so the scatter runs entirely on device (no host roundtrip
# for the scalar fill value or for the Python-list indices).
cache = getattr(self, "_image_tokens_index_cache", None)
if cache is None:
cache = {}
self._image_tokens_index_cache = cache
image_tokens_idx = cache.get(logits.device)
if image_tokens_idx is None:
image_tokens_idx = async_tensor_h2d(
self.model.vocabulary_mapping.image_tokens,
dtype=torch.long,
device=logits.device,
)
cache[logits.device] = image_tokens_idx
logits.index_fill_(1, image_tokens_idx, torch.finfo(logits.dtype).min)
return logits
+13 -4
View File
@@ -54,6 +54,7 @@ from vllm.multimodal.processing.processor import (
)
from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.utils.torch_utils import async_tensor_h2d
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsTranscription
from .utils import (
@@ -683,10 +684,18 @@ class Gemma3nForConditionalGeneration(
# We handle both cases:
# - If fewer tokens: pad with the embedding of the last vocab token
# - If more tokens: truncate to the expected count
# TODO precompute and cache padding
audio_padding_toks = torch.tensor(
[[self.vocab_size - 1]], dtype=torch.long, device=audio_features.device
)
# Cache the single-scalar padding-token tensor per-device to avoid a
# synchronous H2D tensor construction on every forward.
cache = getattr(self, "_audio_padding_toks_cache", None)
if cache is None:
cache = {}
self._audio_padding_toks_cache = cache
audio_padding_toks = cache.get(audio_features.device)
if audio_padding_toks is None:
audio_padding_toks = async_tensor_h2d(
[[self.vocab_size - 1]], dtype=torch.long, device=audio_features.device
)
cache[audio_features.device] = audio_padding_toks
audio_padding_embs = self.embed_audio(input_ids=audio_padding_toks)
audio_features = torch.where(
audio_mask.unsqueeze(-1), audio_padding_embs, audio_features
+11 -4
View File
@@ -99,6 +99,7 @@ from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.processor import get_processor_cls_name_from_config
from vllm.transformers_utils.utils import convert_model_repo_to_path
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers
@@ -535,7 +536,8 @@ class Glm4vVisionEmbeddings(nn.Module):
total_seq = h_coords.shape[0]
device = pos_embed_weight.device
# Move coordinates to correct device
# Move coordinates to correct device. The only caller already hands
# these over pinned + non-blocking, so this is normally a no-op.
h_coords, w_coords = h_coords.to(device), w_coords.to(device)
# Handle empty sequence case
@@ -816,10 +818,15 @@ class Glm4vVisionTransformer(nn.Module):
)
lengths = [h * w] * t
image_shapes = torch.tensor([[t, h, w]], device=device)
image_shapes = async_tensor_h2d(
[[t, h, w]], dtype=torch.long, device=device
)
h_coords_repeated = h_coords.repeat(t)
w_coords_repeated = w_coords.repeat(t)
# Build the coordinates on the host (cheap integer math) but move
# them across pinned + non-blocking, so the consumer's
# `.to(device)` below is a no-op rather than a blocking H2D.
h_coords_repeated = async_tensor_h2d(h_coords.repeat(t), device=device)
w_coords_repeated = async_tensor_h2d(w_coords.repeat(t), device=device)
embeds = self.embeddings(
embeddings=torch.zeros(
@@ -437,7 +437,7 @@ class Qwen3OmniMoeAudioEncoder(nn.Module):
# Compute chunk information
chunk_num = torch.ceil(feature_lens / (self.n_window * 2)).long()
chunk_lengths = torch.tensor(
chunk_lengths = async_tensor_h2d(
[self.n_window * 2] * chunk_num.sum(),
dtype=torch.long,
device=feature_lens.device,
+3 -7
View File
@@ -773,15 +773,11 @@ class VoxtralEncoderModel(nn.Module):
if global_log_mel_max := self.config.global_log_mel_max:
if not isinstance(global_log_mel_max, float):
raise TypeError(f"{global_log_mel_max=} needs to be of type float.")
log_spec_max = torch.tensor(
global_log_mel_max,
device=log_spec.device,
dtype=log_spec.dtype,
)
# Use `clamp` to avoid gpu<->cpu sync.
log_spec = torch.clamp(log_spec, min=global_log_mel_max - 8.0)
else:
log_spec_max = log_spec.max()
log_spec = torch.maximum(log_spec, log_spec_max - 8.0)
log_spec = torch.maximum(log_spec, log_spec_max - 8.0)
log_spec = (log_spec + 4.0) / 4.0
return log_spec.to(input_dtype)