[Perf] Avoid more GPU<->CPU syncs on the model execution path (#51738)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -248,13 +248,13 @@ def _audioflamingo3_field_config(hf_inputs: Mapping[str, torch.Tensor]):
|
||||
feature_attention_mask=MultiModalFieldConfig.flat_from_sizes(
|
||||
"audio", chunk_counts, dim=0
|
||||
),
|
||||
chunk_counts=MultiModalFieldConfig.batched("audio"),
|
||||
chunk_counts=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
|
||||
)
|
||||
return dict(
|
||||
audio_embeds=MultiModalFieldConfig.batched("audio"),
|
||||
input_features=MultiModalFieldConfig.batched("audio"),
|
||||
feature_attention_mask=MultiModalFieldConfig.batched("audio"),
|
||||
chunk_counts=MultiModalFieldConfig.batched("audio"),
|
||||
chunk_counts=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -48,6 +48,7 @@ from vllm.model_executor.models.transformers.utils import recursive_replace_line
|
||||
from vllm.model_executor.models.utils import WeightsMapper, maybe_prefix
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.outputs import LogprobsTensors
|
||||
from vllm.v1.sample.ops.topk_topp_sampler import apply_top_k_top_p
|
||||
from vllm.v1.worker.gpu.attn_utils import build_attn_metadata
|
||||
@@ -728,10 +729,13 @@ class DiffusionGemmaRequestStates:
|
||||
max_num_reqs, canvas_length, hidden_size, dtype=torch.float32, device=device
|
||||
)
|
||||
|
||||
def init_canvas(self, slot_indices_np: np.ndarray) -> None:
|
||||
"""Initialize canvas with random tokens for the given slots."""
|
||||
n = slot_indices_np.shape[0]
|
||||
self.canvas[slot_indices_np] = torch.randint(
|
||||
def init_canvas(self, slot_indices: torch.Tensor) -> None:
|
||||
"""Initialize canvas with random tokens for the given slots.
|
||||
|
||||
`slot_indices` must already be on device to avoid a cpu->gpu sync.
|
||||
"""
|
||||
n = slot_indices.shape[0]
|
||||
self.canvas[slot_indices] = torch.randint(
|
||||
0,
|
||||
self.vocab_size,
|
||||
(n, self.canvas_length),
|
||||
@@ -740,15 +744,15 @@ class DiffusionGemmaRequestStates:
|
||||
)
|
||||
|
||||
def add_request(self, slot_idx: int) -> None:
|
||||
self.is_encoder_phase[slot_idx] = True
|
||||
self.init_canvas(torch.tensor([slot_idx], device=self.device))
|
||||
self.step[slot_idx] = 0
|
||||
self.accepted_canvas_history_len[slot_idx] = 0
|
||||
self.is_encoder_phase[slot_idx].fill_(True)
|
||||
self.init_canvas(async_tensor_h2d([slot_idx], device=self.device))
|
||||
self.step[slot_idx].fill_(0)
|
||||
self.accepted_canvas_history_len[slot_idx].fill_(0)
|
||||
self.self_conditioning_embeds[slot_idx] = 0
|
||||
|
||||
def remove_request(self, slot_idx: int) -> None:
|
||||
self.is_encoder_phase[slot_idx] = False
|
||||
self.accepted_canvas_history_len[slot_idx] = 0
|
||||
self.is_encoder_phase[slot_idx].fill_(False)
|
||||
self.accepted_canvas_history_len[slot_idx].fill_(0)
|
||||
self.self_conditioning_embeds[slot_idx] = 0
|
||||
|
||||
|
||||
@@ -1148,11 +1152,15 @@ class DiffusionSampler:
|
||||
ps = input_batch.idx_mapping_np[prefill_indices_np[done_prefill_np]]
|
||||
if len(ps) == 0:
|
||||
return
|
||||
states.init_canvas(ps)
|
||||
self.req_states.draft_tokens[ps, : self.canvas_length] = states.canvas[ps]
|
||||
# Move the slot indices across once, up front: indexing a device
|
||||
# tensor with a numpy array copies them over synchronously each time.
|
||||
ps_gpu = async_copy_to_gpu(
|
||||
ps.astype(np.int64), device=states.is_encoder_phase.device
|
||||
)
|
||||
states.init_canvas(ps_gpu)
|
||||
self.req_states.draft_tokens[ps_gpu, : self.canvas_length] = states.canvas[
|
||||
ps_gpu
|
||||
]
|
||||
states.is_encoder_phase.index_fill_(0, ps_gpu, False)
|
||||
|
||||
def _handle_prefill(
|
||||
|
||||
@@ -69,6 +69,7 @@ from vllm.multimodal.processing.processor import (
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
from .interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
@@ -2039,7 +2040,20 @@ class Gemma4ForConditionalGeneration(
|
||||
) -> torch.Tensor | None:
|
||||
logits = self.language_model.compute_logits(hidden_states)
|
||||
if logits is not None and self._suppress_token_ids:
|
||||
logits[:, self._suppress_token_ids] = -float("inf")
|
||||
# Cache a per-device index tensor for the (static) suppressed-token
|
||||
# set and use `index_fill_`, so neither the Python-list indices nor
|
||||
# the scalar fill value take a host roundtrip per call.
|
||||
cache = getattr(self, "_suppress_token_ids_cache", None)
|
||||
if cache is None:
|
||||
cache = {}
|
||||
self._suppress_token_ids_cache = cache
|
||||
suppress_idx = cache.get(logits.device)
|
||||
if suppress_idx is None:
|
||||
suppress_idx = async_tensor_h2d(
|
||||
self._suppress_token_ids, dtype=torch.long, device=logits.device
|
||||
)
|
||||
cache[logits.device] = suppress_idx
|
||||
logits.index_fill_(1, suppress_idx, -float("inf"))
|
||||
return logits
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
@@ -548,9 +548,9 @@ class Glm4vVisionEmbeddings(nn.Module):
|
||||
else:
|
||||
# Convert inputs to tensors if needed
|
||||
if isinstance(lengths, list):
|
||||
lengths = torch.tensor(lengths, device=device, dtype=torch.long)
|
||||
lengths = async_tensor_h2d(lengths, device=device, dtype=torch.long)
|
||||
if not isinstance(image_shapes, torch.Tensor):
|
||||
image_shapes = torch.tensor(
|
||||
image_shapes = async_tensor_h2d(
|
||||
image_shapes, device=device, dtype=torch.long
|
||||
)
|
||||
|
||||
|
||||
@@ -426,12 +426,16 @@ class InternS1MultiModalProcessor(BaseMultiModalProcessor[InternS1ProcessingInfo
|
||||
),
|
||||
image_num_patches=MultiModalFieldConfig.batched("image"),
|
||||
image_embeds=MultiModalFieldConfig.batched("image"),
|
||||
image_token_id=MultiModalFieldConfig.shared("image", num_images),
|
||||
image_token_id=MultiModalFieldConfig.shared(
|
||||
"image", num_images, keep_on_cpu=True
|
||||
),
|
||||
pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
|
||||
"video", video_num_patches
|
||||
),
|
||||
video_num_patches=MultiModalFieldConfig.batched("video"),
|
||||
video_token_id=MultiModalFieldConfig.shared("video", num_videos),
|
||||
video_token_id=MultiModalFieldConfig.shared(
|
||||
"video", num_videos, keep_on_cpu=True
|
||||
),
|
||||
)
|
||||
|
||||
def _get_prompt_updates(
|
||||
|
||||
@@ -478,7 +478,9 @@ class InternVLMultiModalProcessor(
|
||||
"video", video_num_patches
|
||||
),
|
||||
video_num_patches=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
|
||||
video_token_id=MultiModalFieldConfig.shared("video", num_videos),
|
||||
video_token_id=MultiModalFieldConfig.shared(
|
||||
"video", num_videos, keep_on_cpu=True
|
||||
),
|
||||
)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
|
||||
@@ -871,12 +871,12 @@ def _keye_field_config(
|
||||
return dict(
|
||||
pixel_values=MultiModalFieldConfig.flat_from_sizes("image", image_grid_sizes),
|
||||
image_embeds=MultiModalFieldConfig.flat_from_sizes("image", image_grid_sizes),
|
||||
image_grid_thw=MultiModalFieldConfig.batched("image"),
|
||||
image_grid_thw=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
|
||||
pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
|
||||
"video", video_grid_sizes
|
||||
),
|
||||
video_embeds=MultiModalFieldConfig.flat_from_sizes("video", video_grid_sizes),
|
||||
video_grid_thw=MultiModalFieldConfig.batched("video"),
|
||||
video_grid_thw=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
|
||||
)
|
||||
|
||||
|
||||
@@ -1291,8 +1291,9 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal):
|
||||
image_grid_thw = image_input["image_grid_thw"]
|
||||
assert image_grid_thw.ndim == 2
|
||||
|
||||
for idx, thaw in enumerate(image_grid_thw):
|
||||
thw_tuple = tuple(thaw.detach().cpu().numpy().tolist())
|
||||
image_grid_thw_list = image_grid_thw.tolist()
|
||||
for idx, thw in enumerate(image_grid_thw_list):
|
||||
thw_tuple = tuple(thw)
|
||||
numel = np.prod(thw_tuple)
|
||||
image_grid_hws.append(thw_tuple)
|
||||
image_position_ids = torch.arange(numel) % np.prod(thw_tuple[1:])
|
||||
|
||||
@@ -266,9 +266,10 @@ class Phi3HDImageEmbedding(nn.Module):
|
||||
)
|
||||
|
||||
batch_image_features_proj = []
|
||||
image_sizes_list = image_sizes.tolist()
|
||||
# need a for loop to process each image because of different image sizes
|
||||
# (patch arrangement is different for each image)
|
||||
for i, img_size in enumerate(image_sizes):
|
||||
for i, img_size in enumerate(image_sizes_list):
|
||||
h, w = img_size
|
||||
h_crop = h // 336
|
||||
w_crop = w // 336
|
||||
@@ -426,7 +427,7 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
|
||||
) -> Mapping[str, MultiModalFieldConfig]:
|
||||
return dict(
|
||||
pixel_values=MultiModalFieldConfig.batched("image"),
|
||||
image_sizes=MultiModalFieldConfig.batched("image"),
|
||||
image_sizes=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
|
||||
image_embeds=MultiModalFieldConfig.batched("image"),
|
||||
)
|
||||
|
||||
|
||||
@@ -270,8 +270,7 @@ class Phi4MMImageEncoder(nn.Module):
|
||||
pixel_values = pixel_values.flatten(0, 1)
|
||||
|
||||
img_features = self.get_img_features(
|
||||
pixel_values,
|
||||
image_attention_mask.type(torch.BoolTensor).flatten(0, 1).to(target_device),
|
||||
pixel_values, image_attention_mask.bool().flatten(0, 1)
|
||||
)
|
||||
|
||||
base_feat_height_target = self.base_feat_height_target
|
||||
|
||||
@@ -200,7 +200,7 @@ class Step3VLMultiModalProcessor(BaseMultiModalProcessor[Step3VLProcessingInfo])
|
||||
patch_pixel_values=MultiModalFieldConfig.flat_from_sizes(
|
||||
"image", num_patches
|
||||
),
|
||||
num_patches=MultiModalFieldConfig.batched("image"),
|
||||
num_patches=MultiModalFieldConfig.batched("image", keep_on_cpu=True),
|
||||
patch_newline_mask=MultiModalFieldConfig.flat_from_sizes(
|
||||
"image", num_patches
|
||||
),
|
||||
@@ -682,7 +682,8 @@ class Step3VLForConditionalGeneration(
|
||||
|
||||
merged_image_features = []
|
||||
cur_patch_idx = 0
|
||||
for i, num_patch in enumerate(num_patches):
|
||||
num_patches_list = num_patches.tolist()
|
||||
for i, num_patch in enumerate(num_patches_list):
|
||||
cur_feature = []
|
||||
if num_patch > 0:
|
||||
patch_slice = patch_image_features[
|
||||
|
||||
Reference in New Issue
Block a user