[Core] Ensure memory is pinned prior to async h2d copy (#45424)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-06-20 20:02:24 -07:00
committed by GitHub
parent 8dd1b702f2
commit 7df3d7dada
49 changed files with 254 additions and 264 deletions
+5 -10
View File
@@ -7,6 +7,7 @@ import numpy as np
import torch
from vllm import SamplingParams
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.sample.logits_processor.interface import (
BatchUpdate,
LogitsProcessor,
@@ -118,7 +119,6 @@ class MinPLogitsProcessor(LogitsProcessor):
class LogitBiasLogitsProcessor(LogitsProcessor):
def __init__(self, _, device: torch.device, is_pin_memory: bool):
self.device = device
self.pin_memory = is_pin_memory
self.biases: dict[int, dict[int, float]] = {}
self.bias_tensor: torch.Tensor = torch.tensor(())
@@ -154,9 +154,7 @@ class LogitBiasLogitsProcessor(LogitsProcessor):
)
def _device_tensor(self, data: list, dtype: torch.dtype) -> torch.Tensor:
return torch.tensor(
data, device="cpu", dtype=dtype, pin_memory=self.pin_memory
).to(device=self.device, non_blocking=True)
return async_tensor_h2d(data, device=self.device, dtype=dtype)
def apply(self, logits: torch.Tensor) -> torch.Tensor:
if self.biases:
@@ -170,7 +168,6 @@ class MinTokensLogitsProcessor(LogitsProcessor):
):
# index -> (min_toks, output_token_ids, stop_token_ids)
self.device = device
self.pin_memory = is_pin_memory
self.min_toks: dict[int, tuple[int, Sequence[int], set[int]]] = {}
# (req_idx_tensor,eos_tok_id_tensor)
@@ -227,9 +224,7 @@ class MinTokensLogitsProcessor(LogitsProcessor):
)
def _device_tensor(self, data: list, dtype: torch.dtype) -> torch.Tensor:
return torch.tensor(
data, device="cpu", dtype=dtype, pin_memory=self.pin_memory
).to(device=self.device, non_blocking=True)
return async_tensor_h2d(data, device=self.device, dtype=dtype)
def apply(self, logits: torch.Tensor) -> torch.Tensor:
if self.min_toks:
@@ -283,8 +278,8 @@ class MinTokensLogitsProcessor(LogitsProcessor):
toks_arr = np.concatenate(all_toks)
# (row_indices, token_indices) for index_put_ to set -inf.
logits_slice = (
torch.from_numpy(rows_arr).to(self.device, non_blocking=True),
torch.from_numpy(toks_arr).to(self.device, non_blocking=True),
async_tensor_h2d(rows_arr, device=self.device),
async_tensor_h2d(toks_arr, device=self.device),
)
logits.index_put_(logits_slice, self.neg_inf_tensor)