[Core] Ensure memory is pinned prior to async h2d copy (#45424)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -10,7 +10,6 @@ from collections.abc import Callable
|
||||
from concurrent.futures import ThreadPoolExecutor, TimeoutError
|
||||
from typing import TYPE_CHECKING, TypeVar
|
||||
|
||||
import numpy as np
|
||||
import regex as re
|
||||
import torch
|
||||
from cachetools import LRUCache
|
||||
@@ -18,7 +17,7 @@ from cachetools import LRUCache
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.import_utils import LazyLoader
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
|
||||
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -123,11 +122,13 @@ def apply_grammar_bitmask(
|
||||
out_indices = []
|
||||
|
||||
# Reorder the bitmask to match the order of the requests in the batch.
|
||||
sorted_bitmask = np.full(
|
||||
shape=(logits.shape[0], grammar_bitmask.shape[1]),
|
||||
fill_value=-1,
|
||||
dtype=grammar_bitmask.dtype,
|
||||
sorted_bitmask_tensor = torch.full(
|
||||
(logits.shape[0], grammar_bitmask.shape[1]),
|
||||
-1,
|
||||
dtype=torch.from_numpy(grammar_bitmask[:0]).dtype,
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
sorted_bitmask = sorted_bitmask_tensor.numpy()
|
||||
cumulative_index = 0
|
||||
for req_id in grammar_output.structured_output_request_ids:
|
||||
num_spec_tokens = len(spec_tokens.get(req_id, ()))
|
||||
@@ -138,10 +139,8 @@ def apply_grammar_bitmask(
|
||||
out_indices.append(bitmask_index)
|
||||
cumulative_index += 1 + num_spec_tokens
|
||||
|
||||
# Copy async to device as tensor.
|
||||
grammar_bitmask = torch.from_numpy(sorted_bitmask).to(
|
||||
logits.device, non_blocking=True
|
||||
)
|
||||
# Copy async to device.
|
||||
grammar_bitmask = sorted_bitmask_tensor.to(logits.device, non_blocking=True)
|
||||
|
||||
# If the length of out indices and the logits have the same shape
|
||||
# we don't need to pass indices to the kernel,
|
||||
@@ -154,11 +153,9 @@ def apply_grammar_bitmask(
|
||||
# xgrammar expects a python list of indices but it will actually work with
|
||||
# a tensor. If we copy the tensor ourselves here we can do it in a
|
||||
# non_blocking manner and there should be no cpu sync within xgrammar.
|
||||
pin_memory = is_pin_memory_available()
|
||||
index_tensor = torch.tensor(
|
||||
out_indices, dtype=torch.int32, device="cpu", pin_memory=pin_memory
|
||||
index_tensor = async_tensor_h2d(
|
||||
out_indices, dtype=torch.int32, device=logits.device
|
||||
)
|
||||
index_tensor = index_tensor.to(logits.device, non_blocking=True)
|
||||
|
||||
xgr.apply_token_bitmask_inplace(logits, grammar_bitmask, indices=index_tensor)
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user