[MRV2] Buffer util simplifications (#56888)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -11,7 +11,7 @@ import torch
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
|
||||
|
||||
@@ -138,7 +138,7 @@ class PPHandler:
|
||||
return None
|
||||
# Filter excluded request indices.
|
||||
idx_mapping_np = np.where(exclude_mask, -1, slot.idx_mapping_np)
|
||||
idx_mapping = async_copy_to_gpu(idx_mapping_np, device=self.device)
|
||||
idx_mapping = async_tensor_h2d(idx_mapping_np, device=self.device)
|
||||
|
||||
self.main_stream.wait_event(slot.event)
|
||||
if slot.draft_tokens is not None and draft_tokens_to_update is not None:
|
||||
@@ -148,7 +148,7 @@ class PPHandler:
|
||||
keep = ~exclude_mask
|
||||
keep_t = torch.as_tensor(keep, device=self.device)
|
||||
draft_tokens = draft_tokens[keep_t]
|
||||
draft_idx_mapping = async_copy_to_gpu(
|
||||
draft_idx_mapping = async_tensor_h2d(
|
||||
slot.idx_mapping_np[keep], device=self.device
|
||||
)
|
||||
draft_tokens_to_update[draft_idx_mapping] = draft_tokens
|
||||
|
||||
Reference in New Issue
Block a user