[Core] Ensure memory is pinned prior to async h2d copy (#45424)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -4,8 +4,7 @@
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.utils import apply_penalties
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
from vllm.utils.torch_utils import make_tensor_with_pad
|
||||
from vllm.utils.torch_utils import PIN_MEMORY, make_tensor_with_pad
|
||||
|
||||
|
||||
def apply_all_penalties(
|
||||
@@ -52,6 +51,6 @@ def _convert_to_tensors(
|
||||
pad=vocab_size,
|
||||
device="cpu",
|
||||
dtype=torch.int64,
|
||||
pin_memory=is_pin_memory_available(),
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
return output_tokens_tensor.to(device, non_blocking=True)
|
||||
|
||||
Reference in New Issue
Block a user