[Core] Ensure memory is pinned prior to async h2d copy (#45424)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-06-20 20:02:24 -07:00
committed by GitHub
parent 8dd1b702f2
commit 7df3d7dada
49 changed files with 254 additions and 264 deletions
+11 -14
View File
@@ -10,7 +10,6 @@ from collections.abc import Callable
from concurrent.futures import ThreadPoolExecutor, TimeoutError
from typing import TYPE_CHECKING, TypeVar
import numpy as np
import regex as re
import torch
from cachetools import LRUCache
@@ -18,7 +17,7 @@ from cachetools import LRUCache
import vllm.envs as envs
from vllm.logger import init_logger
from vllm.utils.import_utils import LazyLoader
from vllm.utils.platform_utils import is_pin_memory_available
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
if TYPE_CHECKING:
@@ -123,11 +122,13 @@ def apply_grammar_bitmask(
out_indices = []
# Reorder the bitmask to match the order of the requests in the batch.
sorted_bitmask = np.full(
shape=(logits.shape[0], grammar_bitmask.shape[1]),
fill_value=-1,
dtype=grammar_bitmask.dtype,
sorted_bitmask_tensor = torch.full(
(logits.shape[0], grammar_bitmask.shape[1]),
-1,
dtype=torch.from_numpy(grammar_bitmask[:0]).dtype,
pin_memory=PIN_MEMORY,
)
sorted_bitmask = sorted_bitmask_tensor.numpy()
cumulative_index = 0
for req_id in grammar_output.structured_output_request_ids:
num_spec_tokens = len(spec_tokens.get(req_id, ()))
@@ -138,10 +139,8 @@ def apply_grammar_bitmask(
out_indices.append(bitmask_index)
cumulative_index += 1 + num_spec_tokens
# Copy async to device as tensor.
grammar_bitmask = torch.from_numpy(sorted_bitmask).to(
logits.device, non_blocking=True
)
# Copy async to device.
grammar_bitmask = sorted_bitmask_tensor.to(logits.device, non_blocking=True)
# If the length of out indices and the logits have the same shape
# we don't need to pass indices to the kernel,
@@ -154,11 +153,9 @@ def apply_grammar_bitmask(
# xgrammar expects a python list of indices but it will actually work with
# a tensor. If we copy the tensor ourselves here we can do it in a
# non_blocking manner and there should be no cpu sync within xgrammar.
pin_memory = is_pin_memory_available()
index_tensor = torch.tensor(
out_indices, dtype=torch.int32, device="cpu", pin_memory=pin_memory
index_tensor = async_tensor_h2d(
out_indices, dtype=torch.int32, device=logits.device
)
index_tensor = index_tensor.to(logits.device, non_blocking=True)
xgr.apply_token_bitmask_inplace(logits, grammar_bitmask, indices=index_tensor)
return