[MRV2] Buffer util simplifications (#56888)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -19,6 +19,7 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
triton_kernel_dispatcher_with_warmup,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.outputs import LogprobsLists, LogprobsTensors, SamplerOutput
|
||||
from vllm.v1.sample.logits_processor.builtin import MinTokensLogitsProcessor
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
@@ -327,9 +328,7 @@ class RejectionSampler(nn.Module):
|
||||
num_draft_tokens = torch.tensor(metadata.num_draft_tokens, device="cpu")
|
||||
original_indices = torch.arange(num_requests, device="cpu")
|
||||
repeat_indices_cpu = original_indices.repeat_interleave(num_draft_tokens)
|
||||
repeat_indices = repeat_indices_cpu.to(
|
||||
device=logits.device, non_blocking=True
|
||||
)
|
||||
repeat_indices = async_tensor_h2d(repeat_indices_cpu, logits.device)
|
||||
logits = self.apply_penalties(
|
||||
logits, sampling_metadata, metadata, repeat_indices, output_token_ids
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user