[Perf] Avoid h2d copies from non-pinned CPU tensors (#54299)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-08-29 12:06:30 +08:00
committed by GitHub
parent 5182f2705b
commit 6c18a54648
14 changed files with 81 additions and 84 deletions
+3 -6
View File
@@ -32,6 +32,7 @@ from vllm.model_executor.models.phi4mm_utils import (
get_offset,
unfold_tensor,
)
from vllm.utils.torch_utils import async_tensor_h2d
class ConformerEncoderLayer(nn.Module):
@@ -586,9 +587,7 @@ class TransformerEncoderBase(abc.ABC, nn.Module):
seq_len, batch_size, self.chunk_size, self.left_chunk
)
device = xs_pad.device
enc_streaming_mask = enc_streaming_mask.contiguous().to(
device, non_blocking=True
)
enc_streaming_mask = async_tensor_h2d(enc_streaming_mask.contiguous(), device)
xs_pad = xs_pad.to(device)
input_tensor = xs_pad
@@ -921,9 +920,7 @@ class ConformerEncoder(TransformerEncoderBase):
enc_streaming_mask = self._streaming_mask(
max_audio_length, batch_size, self.chunk_size, self.left_chunk
)
enc_streaming_mask = enc_streaming_mask.contiguous().to(
device, non_blocking=True
)
enc_streaming_mask = async_tensor_h2d(enc_streaming_mask.contiguous(), device)
if mask is None:
return enc_streaming_mask