[Perf] Avoid some more unnecessary GPU<->CPU syncs (#51458)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-08-08 17:13:26 -07:00
committed by GitHub
parent d608dfabfd
commit 9b0afeb4f6
13 changed files with 99 additions and 49 deletions
+14 -6
View File
@@ -14,6 +14,7 @@ from vllm.config import VllmConfig
from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger
from vllm.sampling_params import SamplingParams
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.sample.logits_processor import (
LOGITSPROCS_GROUP,
AdapterLogitsProcessor,
@@ -90,17 +91,24 @@ class DummyLogitsProcessor(LogitsProcessor):
if not self.req_info:
return logits
# Save target values before modification
cols = torch.tensor(
# Save target values before modification.
cols = async_tensor_h2d(
list(self.req_info.values()), dtype=torch.long, device=logits.device
)
rows = torch.tensor(
rows = async_tensor_h2d(
list(self.req_info.keys()), dtype=torch.long, device=logits.device
)
values_to_keep = logits[rows, cols].clone()
# Mask all but target tokens
logits[rows] = float("-inf")
# Mask all but target tokens. Use an on-device fill tensor so the
# scatter doesn't force a synchronizing scalar H2D.
fill = torch.full(
(rows.numel(), logits.size(-1)),
float("-inf"),
dtype=logits.dtype,
device=logits.device,
)
logits[rows] = fill
logits[rows, cols] = values_to_keep
return logits
@@ -141,7 +149,7 @@ class DummyPerReqLogitsProcessor:
output_ids: list[int],
logits: torch.Tensor,
) -> torch.Tensor:
val_to_keep = logits[self.target_token].item()
val_to_keep = logits[self.target_token].clone()
logits[:] = float("-inf")
logits[self.target_token] = val_to_keep
return logits