[Perf] Avoid some more unnecessary GPU<->CPU syncs (#51458)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -99,6 +99,7 @@ from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.processor import get_processor_cls_name_from_config
|
||||
from vllm.transformers_utils.utils import convert_model_repo_to_path
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers
|
||||
|
||||
@@ -535,7 +536,8 @@ class Glm4vVisionEmbeddings(nn.Module):
|
||||
total_seq = h_coords.shape[0]
|
||||
device = pos_embed_weight.device
|
||||
|
||||
# Move coordinates to correct device
|
||||
# Move coordinates to correct device. The only caller already hands
|
||||
# these over pinned + non-blocking, so this is normally a no-op.
|
||||
h_coords, w_coords = h_coords.to(device), w_coords.to(device)
|
||||
|
||||
# Handle empty sequence case
|
||||
@@ -816,10 +818,15 @@ class Glm4vVisionTransformer(nn.Module):
|
||||
)
|
||||
|
||||
lengths = [h * w] * t
|
||||
image_shapes = torch.tensor([[t, h, w]], device=device)
|
||||
image_shapes = async_tensor_h2d(
|
||||
[[t, h, w]], dtype=torch.long, device=device
|
||||
)
|
||||
|
||||
h_coords_repeated = h_coords.repeat(t)
|
||||
w_coords_repeated = w_coords.repeat(t)
|
||||
# Build the coordinates on the host (cheap integer math) but move
|
||||
# them across pinned + non-blocking, so the consumer's
|
||||
# `.to(device)` below is a no-op rather than a blocking H2D.
|
||||
h_coords_repeated = async_tensor_h2d(h_coords.repeat(t), device=device)
|
||||
w_coords_repeated = async_tensor_h2d(w_coords.repeat(t), device=device)
|
||||
|
||||
embeds = self.embeddings(
|
||||
embeddings=torch.zeros(
|
||||
|
||||
Reference in New Issue
Block a user