[Perf] Avoid more GPU<->CPU syncs on the model execution path (#51738)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-08-11 19:30:34 -07:00
committed by GitHub
parent 23f25aac14
commit f97e502969
15 changed files with 116 additions and 56 deletions
+2 -2
View File
@@ -248,13 +248,13 @@ def _audioflamingo3_field_config(hf_inputs: Mapping[str, torch.Tensor]):
feature_attention_mask=MultiModalFieldConfig.flat_from_sizes(
"audio", chunk_counts, dim=0
),
chunk_counts=MultiModalFieldConfig.batched("audio"),
chunk_counts=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
)
return dict(
audio_embeds=MultiModalFieldConfig.batched("audio"),
input_features=MultiModalFieldConfig.batched("audio"),
feature_attention_mask=MultiModalFieldConfig.batched("audio"),
chunk_counts=MultiModalFieldConfig.batched("audio"),
chunk_counts=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
)
+20 -12
View File
@@ -48,6 +48,7 @@ from vllm.model_executor.models.transformers.utils import recursive_replace_line
from vllm.model_executor.models.utils import WeightsMapper, maybe_prefix
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.platforms import current_platform
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.outputs import LogprobsTensors
from vllm.v1.sample.ops.topk_topp_sampler import apply_top_k_top_p
from vllm.v1.worker.gpu.attn_utils import build_attn_metadata
@@ -728,10 +729,13 @@ class DiffusionGemmaRequestStates:
max_num_reqs, canvas_length, hidden_size, dtype=torch.float32, device=device
)
def init_canvas(self, slot_indices_np: np.ndarray) -> None:
"""Initialize canvas with random tokens for the given slots."""
n = slot_indices_np.shape[0]
self.canvas[slot_indices_np] = torch.randint(
def init_canvas(self, slot_indices: torch.Tensor) -> None:
"""Initialize canvas with random tokens for the given slots.
`slot_indices` must already be on device to avoid a cpu->gpu sync.
"""
n = slot_indices.shape[0]
self.canvas[slot_indices] = torch.randint(
0,
self.vocab_size,
(n, self.canvas_length),
@@ -740,15 +744,15 @@ class DiffusionGemmaRequestStates:
)
def add_request(self, slot_idx: int) -> None:
self.is_encoder_phase[slot_idx] = True
self.init_canvas(torch.tensor([slot_idx], device=self.device))
self.step[slot_idx] = 0
self.accepted_canvas_history_len[slot_idx] = 0
self.is_encoder_phase[slot_idx].fill_(True)
self.init_canvas(async_tensor_h2d([slot_idx], device=self.device))
self.step[slot_idx].fill_(0)
self.accepted_canvas_history_len[slot_idx].fill_(0)
self.self_conditioning_embeds[slot_idx] = 0
def remove_request(self, slot_idx: int) -> None:
self.is_encoder_phase[slot_idx] = False
self.accepted_canvas_history_len[slot_idx] = 0
self.is_encoder_phase[slot_idx].fill_(False)
self.accepted_canvas_history_len[slot_idx].fill_(0)
self.self_conditioning_embeds[slot_idx] = 0
@@ -1148,11 +1152,15 @@ class DiffusionSampler:
ps = input_batch.idx_mapping_np[prefill_indices_np[done_prefill_np]]
if len(ps) == 0:
return
states.init_canvas(ps)
self.req_states.draft_tokens[ps, : self.canvas_length] = states.canvas[ps]
# Move the slot indices across once, up front: indexing a device
# tensor with a numpy array copies them over synchronously each time.
ps_gpu = async_copy_to_gpu(
ps.astype(np.int64), device=states.is_encoder_phase.device
)
states.init_canvas(ps_gpu)
self.req_states.draft_tokens[ps_gpu, : self.canvas_length] = states.canvas[
ps_gpu
]
states.is_encoder_phase.index_fill_(0, ps_gpu, False)
def _handle_prefill(
+15 -1
View File
@@ -69,6 +69,7 @@ from vllm.multimodal.processing.processor import (
)
from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.utils.torch_utils import async_tensor_h2d
from .interfaces import (
MultiModalEmbeddings,
@@ -2039,7 +2040,20 @@ class Gemma4ForConditionalGeneration(
) -> torch.Tensor | None:
logits = self.language_model.compute_logits(hidden_states)
if logits is not None and self._suppress_token_ids:
logits[:, self._suppress_token_ids] = -float("inf")
# Cache a per-device index tensor for the (static) suppressed-token
# set and use `index_fill_`, so neither the Python-list indices nor
# the scalar fill value take a host roundtrip per call.
cache = getattr(self, "_suppress_token_ids_cache", None)
if cache is None:
cache = {}
self._suppress_token_ids_cache = cache
suppress_idx = cache.get(logits.device)
if suppress_idx is None:
suppress_idx = async_tensor_h2d(
self._suppress_token_ids, dtype=torch.long, device=logits.device
)
cache[logits.device] = suppress_idx
logits.index_fill_(1, suppress_idx, -float("inf"))
return logits
# ------------------------------------------------------------------ #
+2 -2
View File
@@ -548,9 +548,9 @@ class Glm4vVisionEmbeddings(nn.Module):
else:
# Convert inputs to tensors if needed
if isinstance(lengths, list):
lengths = torch.tensor(lengths, device=device, dtype=torch.long)
lengths = async_tensor_h2d(lengths, device=device, dtype=torch.long)
if not isinstance(image_shapes, torch.Tensor):
image_shapes = torch.tensor(
image_shapes = async_tensor_h2d(
image_shapes, device=device, dtype=torch.long
)
+6 -2
View File
@@ -426,12 +426,16 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
),
image_num_patches=MultiModalFieldConfig.batched("image"),
image_embeds=MultiModalFieldConfig.batched("image"),
image_token_id=MultiModalFieldConfig.shared("image", num_images),
image_token_id=MultiModalFieldConfig.shared(
"image", num_images, keep_on_cpu=True
),
pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
"video", video_num_patches
),
video_num_patches=MultiModalFieldConfig.batched("video"),
video_token_id=MultiModalFieldConfig.shared("video", num_videos),
video_token_id=MultiModalFieldConfig.shared(
"video", num_videos, keep_on_cpu=True
),
)
def _get_prompt_updates(
+3 -1
View File
@@ -478,7 +478,9 @@ class InternVLMultiModalProcessor(
"video", video_num_patches
),
video_num_patches=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
video_token_id=MultiModalFieldConfig.shared("video", num_videos),
video_token_id=MultiModalFieldConfig.shared(
"video", num_videos, keep_on_cpu=True
),
)
def _get_mm_fields_config(
+5 -4
View File
@@ -871,12 +871,12 @@ def _keye_field_config(
return dict(
pixel_values=MultiModalFieldConfig.flat_from_sizes("image", image_grid_sizes),
image_embeds=MultiModalFieldConfig.flat_from_sizes("image", image_grid_sizes),
image_grid_thw=MultiModalFieldConfig.batched("image"),
image_grid_thw=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
"video", video_grid_sizes
),
video_embeds=MultiModalFieldConfig.flat_from_sizes("video", video_grid_sizes),
video_grid_thw=MultiModalFieldConfig.batched("video"),
video_grid_thw=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
)
@@ -1291,8 +1291,9 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal):
image_grid_thw = image_input["image_grid_thw"]
assert image_grid_thw.ndim == 2
for idx, thaw in enumerate(image_grid_thw):
thw_tuple = tuple(thaw.detach().cpu().numpy().tolist())
image_grid_thw_list = image_grid_thw.tolist()
for idx, thw in enumerate(image_grid_thw_list):
thw_tuple = tuple(thw)
numel = np.prod(thw_tuple)
image_grid_hws.append(thw_tuple)
image_position_ids = torch.arange(numel) % np.prod(thw_tuple[1:])
+3 -2
View File
@@ -266,9 +266,10 @@ class Phi3HDImageEmbedding(nn.Module):
)
batch_image_features_proj = []
image_sizes_list = image_sizes.tolist()
# need a for loop to process each image because of different image sizes
# (patch arrangement is different for each image)
for i, img_size in enumerate(image_sizes):
for i, img_size in enumerate(image_sizes_list):
h, w = img_size
h_crop = h // 336
w_crop = w // 336
@@ -426,7 +427,7 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
) -> Mapping[str, MultiModalFieldConfig]:
return dict(
pixel_values=MultiModalFieldConfig.batched("image"),
image_sizes=MultiModalFieldConfig.batched("image"),
image_sizes=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
image_embeds=MultiModalFieldConfig.batched("image"),
)
+1 -2
View File
@@ -270,8 +270,7 @@ class Phi4MMImageEncoder(nn.Module):
pixel_values = pixel_values.flatten(0, 1)
img_features = self.get_img_features(
pixel_values,
image_attention_mask.type(torch.BoolTensor).flatten(0, 1).to(target_device),
pixel_values, image_attention_mask.bool().flatten(0, 1)
)
base_feat_height_target = self.base_feat_height_target
+3 -2
View File
@@ -200,7 +200,7 @@ class Step3VLMultiModalProcessor(BaseMultiModalProcessor[Step3VLProcessingInfo])
patch_pixel_values=MultiModalFieldConfig.flat_from_sizes(
"image", num_patches
),
num_patches=MultiModalFieldConfig.batched("image"),
num_patches=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
patch_newline_mask=MultiModalFieldConfig.flat_from_sizes(
"image", num_patches
),
@@ -682,7 +682,8 @@ class Step3VLForConditionalGeneration(
merged_image_features = []
cur_patch_idx = 0
for i, num_patch in enumerate(num_patches):
num_patches_list = num_patches.tolist()
for i, num_patch in enumerate(num_patches_list):
cur_feature = []
if num_patch > 0:
patch_slice = patch_image_features[